home *** CD-ROM | disk | FTP | other *** search
/ Chip 2004 July / CMCD0704.ISO / Software / Freeware / Comunicatii / htttrack / httrack-3.32-2.exe / {app} / src / htsback.c < prev    next >
Encoding:
C/C++ Source or Header  |  2004-05-08  |  116.1 KB  |  2,977 lines
  1. /* ------------------------------------------------------------ */
  2. /*
  3. HTTrack Website Copier, Offline Browser for Windows and Unix
  4. Copyright (C) Xavier Roche and other contributors
  5.  
  6. This program is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU General Public License
  8. as published by the Free Software Foundation; either version 2
  9. of the License, or any later version.
  10.  
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. GNU General Public License for more details.
  15.  
  16. You should have received a copy of the GNU General Public License
  17. along with this program; if not, write to the Free Software
  18. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19.  
  20.  
  21. Important notes:
  22.  
  23. - We hereby ask people using this source NOT to use it in purpose of grabbing
  24. emails addresses, or collecting any other private information on persons.
  25. This would disgrace our work, and spoil the many hours we spent on it.
  26.  
  27.  
  28. Please visit our Website: http://www.httrack.com
  29. */
  30.  
  31.  
  32. /* ------------------------------------------------------------ */
  33. /* File: httrack.c subroutines:                                 */
  34. /*       backing system (multiple socket download)              */
  35. /* Author: Xavier Roche                                         */
  36. /* ------------------------------------------------------------ */
  37.  
  38. /* Internal engine bytecode */
  39. #define HTS_INTERNAL_BYTECODE
  40.  
  41. #include "htsback.h"
  42.  
  43. /* specific definitions */
  44. #include "htsbase.h"
  45. #include "htsnet.h"
  46. #include "htsthread.h"
  47. #include <time.h>
  48. /* END specific definitions */
  49.  
  50. //#if HTS_WIN
  51. #include "htsftp.h"
  52. #if HTS_USEZLIB
  53. #include "htszlib.h"
  54. #else
  55. #error HTS_USEZLIB not defined
  56. #endif
  57. //#endif
  58.  
  59. #if HTS_WIN
  60. #ifndef __cplusplus
  61. // DOS
  62. #ifndef  _WIN32_WCE
  63. #include <process.h>    /* _beginthread, _endthread */
  64. #endif
  65. #endif
  66. #else
  67. #endif
  68.  
  69. #undef test_flush
  70. #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->errlog) { fflush(opt->errlog);  } }
  71.  
  72. #define VT_CLREOL       "\33[K"
  73.  
  74.  
  75. // ---
  76. // routines de backing
  77. // retourne l'index d'un lien dans un tableau de backing
  78. int back_index(lien_back* back,int back_max,char* adr,char* fil,char* sav) {
  79.   int i=0;
  80.   int index=-1;
  81.   while( i<back_max ) {
  82.     if (back[i].status>=0)    // rΘception OU prΩt
  83.       if (strfield2(back[i].url_adr,adr)) {
  84.         if (strcmp(back[i].url_fil,fil)==0) {
  85.           if (index==-1)    /* first time we meet, store it */
  86.             index=i;
  87.           else if (strcmp(back[i].url_sav,sav)==0) {  /* oops, check sav too */
  88.             index=i;
  89.             return index;
  90.           }
  91.         }
  92.       }
  93.     i++;
  94.   }
  95.   return index;
  96. }
  97.  
  98. // nombre d'entrΘes libres dans le backing
  99. int back_available(lien_back* back,int back_max) {
  100.   int i;
  101.   int nb=0;
  102.   for(i=0;i<back_max;i++)
  103.     if (back[i].status==-1)     /* libre */
  104.       nb++;
  105.   return nb;
  106. }
  107.  
  108. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  109. LLint back_incache(lien_back* back,int back_max) {
  110.   int i;
  111.   LLint sum=0;
  112.   for(i=0;i<back_max;i++)
  113.     if (back[i].status!=-1)
  114.       if (back[i].r.adr)       // ne comptabilier que les blocs en mΘmoire
  115.         sum+=max(back[i].r.size,back[i].r.totalsize);
  116.   return sum;
  117. }
  118.  
  119. // le lien a-t-il ΘtΘ mis en backing?
  120. HTS_INLINE int back_exist(lien_back* back,int back_max,char* adr,char* fil,char* sav) {
  121.   return (back_index(back,back_max,adr,fil,sav)>=0);
  122. }
  123.  
  124. // nombre de sockets en tΓche de fond
  125. int back_nsoc(lien_back* back,int back_max) {
  126.   int n=0;
  127.   int i;
  128.   for(i=0;i<back_max;i++)
  129.     if (back[i].status > 0)    // only receive
  130.       n++;
  131.  
  132.   return n;
  133. }
  134. int back_nsoc_overall(lien_back* back,int back_max) {
  135.   int n=0;
  136.   int i;
  137.   for(i=0;i<back_max;i++)
  138.     if (back[i].status > 0 || back[i].status == -103)
  139.       n++;
  140.  
  141.   return n;
  142. }
  143.  
  144. // objet (lien) tΘlΘchargΘ ou transfΘrΘ depuis le cache
  145. //
  146. // fermer les paramΦtres de transfert,
  147. // et notamment vΘrifier les fichiers compressΘs (dΘcompresser), callback etc.
  148. int back_finalize(httrackp* opt,cache_back* cache,lien_back* back,int p) {
  149.   if (!back[p].finalized) {
  150.     back[p].finalized = 1;
  151.     if (
  152.       (back[p].status == 0)      // ready
  153.       &&
  154.       (back[p].r.statuscode>0)   // not internal error
  155.       ) {
  156.       if (!back[p].testmode) {        // not test mode
  157.       char* state="unknown";
  158.       
  159.       /* dΘcompression */
  160. #if HTS_USEZLIB
  161.       if (gz_is_available && back[p].r.compressed) {
  162.         if (back[p].r.size > 0) {
  163.           //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
  164.           // stats
  165.           back[p].compressed_size=back[p].r.size;
  166.           // en mΘmoire -> passage sur disque
  167.           if (!back[p].r.is_write) {
  168.             back[p].tmpfile_buffer[0]='\0';
  169.             back[p].tmpfile=tmpnam(back[p].tmpfile_buffer);
  170.             if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  171.               back[p].r.out=fopen(back[p].tmpfile,"wb");
  172.               if (back[p].r.out) {
  173.                 if ((back[p].r.adr) && (back[p].r.size>0)) {
  174.                   if (fwrite(back[p].r.adr,1,(INTsys)back[p].r.size,back[p].r.out) != back[p].r.size) {
  175.                     back[p].r.statuscode=-1;
  176.                     strcpybuff(back[p].r.msg,"Write error when decompressing");
  177.                   }
  178.                 } else {
  179.                   back[p].tmpfile[0]='\0';
  180.                   back[p].r.statuscode=-1;
  181.                   strcpybuff(back[p].r.msg,"Empty compressed file");
  182.                 }
  183.               } else {
  184.                 back[p].tmpfile[0]='\0';
  185.                 back[p].r.statuscode=-1;
  186.                 strcpybuff(back[p].r.msg,"Open error when decompressing");
  187.               }
  188.             }
  189.           }
  190.           // fermer fichier sortie
  191.           if (back[p].r.out!=NULL) {
  192.             fclose(back[p].r.out);
  193.             back[p].r.out=NULL;
  194.           }
  195.           // dΘcompression
  196.           if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0' && back[p].url_sav[0]) {
  197.             LLint size;
  198.             filecreateempty(back[p].url_sav);      // filenote & co
  199.             if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) {
  200.               back[p].r.size=back[p].r.totalsize=size;
  201.               // fichier -> mΘmoire
  202.               if (!back[p].r.is_write) {
  203.                 deleteaddr(&back[p].r);
  204.                 back[p].r.adr=readfile(back[p].url_sav);
  205.                 if (!back[p].r.adr) {
  206.                   back[p].r.statuscode=-1;
  207.                   strcpybuff(back[p].r.msg,"Read error when decompressing");
  208.                 }
  209.                 remove(back[p].url_sav);
  210.               }
  211.             }
  212.             remove(back[p].tmpfile);
  213.           }
  214.           // stats
  215.           HTS_STAT.total_packed+=back[p].compressed_size;
  216.           HTS_STAT.total_unpacked+=back[p].r.size;
  217.           HTS_STAT.total_packedfiles++;
  218.           // unflag
  219.         }
  220.       }
  221.       back[p].r.compressed=0;
  222. #endif
  223.       
  224.       /* Stats */
  225.       if (cache->txt) {
  226.         char flags[32];
  227.         char s[256];
  228.         time_t tt;
  229.         struct tm* A;
  230.         tt=time(NULL);
  231.         A=localtime(&tt);
  232.         if (A == NULL) {
  233.           int localtime_returned_null=0;
  234.           assert(localtime_returned_null);
  235.         }
  236.         strftime(s,250,"%H:%M:%S",A);
  237.         
  238.         flags[0]='\0';
  239.         /* input flags */
  240.         if (back[p].is_update)
  241.           strcatbuff(flags, "U");   // update request
  242.         else
  243.           strcatbuff(flags, "-");
  244.         if (back[p].range_req_size)
  245.           strcatbuff(flags, "R");   // range request
  246.         else
  247.           strcatbuff(flags, "-");
  248.         /* state flags */
  249.         if (back[p].r.is_file)  // direct to disk
  250.           strcatbuff(flags, "F");
  251.         else
  252.           strcatbuff(flags, "-");
  253.         /* output flags */
  254.         if (!back[p].r.notmodified)
  255.           strcatbuff(flags, "M");   // modified
  256.         else
  257.           strcatbuff(flags, "-");
  258.         if (back[p].r.is_chunk)  // chunked
  259.           strcatbuff(flags, "C");
  260.         else
  261.           strcatbuff(flags, "-");
  262.         if (back[p].r.compressed)
  263.           strcatbuff(flags, "Z");   // gzip
  264.         else
  265.           strcatbuff(flags, "-");
  266.         /* Err I had to split these.. */
  267.         fprintf(cache->txt,"%s\t", s);
  268.         fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size);
  269.         fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize);
  270.         fprintf(cache->txt,"\t%s\t",flags);
  271.       }
  272.       if (back[p].r.statuscode==200) {
  273.         if (back[p].r.size>=0) {
  274.           if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) {
  275.             HTS_STAT.stat_bytes+=back[p].r.size;
  276.             HTS_STAT.stat_files++;
  277.           }
  278.           if ( (!back[p].r.notmodified) && (opt->is_update) ) { 
  279.             HTS_STAT.stat_updated_files++;       // page modifiΘe
  280.             if (opt->log!=NULL) {
  281.               fspc(opt->log,"info");
  282.               if (back[p].is_update) {
  283.                 fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  284.               } else {
  285.                 fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  286.               }
  287.               test_flush;
  288.             }
  289.             if (cache->txt) {
  290.               if (back[p].is_update) {
  291.                 state="updated";
  292.               } else {
  293.                 state="added";
  294.               }
  295.             }
  296.           } else {
  297.             if ( (opt->debug>0) && (opt->log!=NULL) ) {
  298.               fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  299.               test_flush;
  300.             }
  301.             if (cache->txt) {
  302.               if (opt->is_update)
  303.                 state="untouched";
  304.               else
  305.                 state="added";
  306.             }
  307.           }
  308.         } else {
  309.           if ( (opt->debug>0) && (opt->log!=NULL) ) {
  310.             fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  311.             test_flush;
  312.           }
  313.           if (cache->txt) {
  314.             state="empty";
  315.           }
  316.         }
  317.       } else {
  318.         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  319.           fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  320.         }
  321.         if (cache->txt) {
  322.           state="error";
  323.         }
  324.       }
  325.       if (cache->txt) {
  326.         fprintf(cache->txt,
  327.           "%d\t"
  328.           "%s ('%s')\t"
  329.           "%s\t"
  330.           "%s%s\t"
  331.           "%s%s\t%s\t"
  332.           "(from %s%s)"
  333.           LF,
  334.           back[p].r.statuscode,
  335.           state, escape_check_url_addr(back[p].r.msg),
  336.           escape_check_url_addr(back[p].r.contenttype),
  337.           ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr((back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)),
  338.           escape_check_url_addr(back[p].url_adr),escape_check_url_addr(back[p].url_fil),escape_check_url_addr(back[p].url_sav),
  339.           escape_check_url_addr(back[p].referer_adr),escape_check_url_addr(back[p].referer_fil)
  340.           );
  341.         if (opt->flush)
  342.           fflush(cache->txt);
  343.       }
  344.       
  345.       /* Cache */
  346.       cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  347.       
  348.       // status finished callback
  349. #if HTS_ANALYSTE
  350.       hts_htmlcheck_xfrstatus(&back[p]);
  351. #endif
  352.       return 0;
  353.   } else {        // testmode
  354.     if (back[p].r.statuscode / 100 >= 3) {        /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */
  355.       /* Cache */
  356.       cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL);
  357.     }
  358.   }
  359.   }
  360.   }
  361.   return -1;
  362. }
  363.  
  364. /* try to keep the connection alive */
  365. int back_letlive(httrackp* opt, cache_back* cache, lien_back* back, int p) {
  366.   int checkerror;
  367.   htsblk* src = &back[p].r;
  368.   if (src && !src->is_file 
  369.     && src->soc != INVALID_SOCKET
  370.     && src->statuscode >= 0           /* no timeout errors & co */
  371.     && src->keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  372.     && ! ( checkerror = check_sockerror(src->soc) )
  373.     /*&& !check_sockdata(src->soc)*/     /* no unexpected data */
  374.     ) {
  375.     htsblk tmp;
  376.     memset(&tmp, 0, sizeof(tmp));
  377.     /* clear everything but connection: switch, close, and reswitch */
  378.     back_connxfr(src, &tmp);
  379.     back_delete(opt, cache, back, p);
  380.     //deletehttp(src);
  381.     back_connxfr(&tmp, src);
  382.     src->req.flush_garbage=1;     /* ignore CRLF garbage */
  383.     return 1;
  384.   }
  385.   return 0;
  386. }
  387.  
  388. void back_connxfr(htsblk* src, htsblk* dst) {
  389.   dst->soc = src->soc;
  390.   src->soc = INVALID_SOCKET;
  391. #if HTS_USEOPENSSL
  392.   dst->ssl = src->ssl;
  393.   src->ssl = 0;
  394.   dst->ssl_con = src->ssl_con;
  395.   src->ssl_con = NULL;
  396. #endif
  397.   dst->keep_alive = src->keep_alive;
  398.   src->keep_alive = 0;
  399.   dst->keep_alive_max = src->keep_alive_max;
  400.   src->keep_alive_max = 0;
  401.   dst->keep_alive_t = src->keep_alive_t;
  402.   src->keep_alive_t = 0;
  403.   dst->debugid = src->debugid;
  404.   src->debugid = 0;
  405. }
  406.  
  407. // clear, or leave for keep-alive
  408. int back_maydelete(httrackp* opt,cache_back* cache,lien_back* back, int p) {
  409.   if (p>=0) {    // on sait jamais..
  410.     if (!opt->nokeepalive
  411.       && back[p].r.keep_alive 
  412.       && back[p].r.keep_alive_max > 1
  413.       && back[p].ka_time_start 
  414.       && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t
  415.       ) {
  416.       lien_back tmp;
  417.       strcpybuff(tmp.url_adr, back[p].url_adr);
  418.       if (back_letlive(opt, cache, back, p)) {
  419.         strcpybuff(back[p].url_adr, tmp.url_adr);
  420.         back[p].status = -103;  // alive & waiting
  421.         if ((opt->debug>1) && (opt->log!=NULL)) {
  422.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully saved #%d (%s)"LF, 
  423.             back[p].r.debugid,
  424.             back[p].url_adr); test_flush;
  425.         }
  426.         return 1;
  427.       }
  428.     }
  429.     back_delete(opt,cache,back, p);
  430.   }
  431.   return 0;
  432. }
  433.  
  434. // clear, or leave for keep-alive
  435. void back_maydeletehttp(httrackp* opt, cache_back* cache, lien_back* back, int back_max, int p) {
  436.   TStamp lt = 0;
  437.   if (back[p].r.soc!=INVALID_SOCKET) {
  438.     int q;
  439.     if (
  440.       back[p].r.soc != INVALID_SOCKET
  441.       && back[p].r.statuscode >= 0           /* no timeout errors & co */
  442.       && back[p].r.keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  443.       && !check_sockerror(back[p].r.soc)
  444.       /* */
  445.       && !opt->nokeepalive
  446.       && back[p].r.keep_alive 
  447.       && back[p].r.keep_alive_max > 1
  448.       && back[p].ka_time_start 
  449.       && ( lt = time_local() ) < back[p].ka_time_start + back[p].r.keep_alive_t
  450.       && ( q = back_search(opt, cache, back, back_max) ) >= 0
  451.       ) 
  452.     {
  453.       lien_back tmp;
  454.       strcpybuff(tmp.url_adr, back[p].url_adr);
  455.       deletehttp(&back[q].r);               // security check
  456.       back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q
  457.       back[q].ka_time_start = back[p].ka_time_start;  // refresh
  458.       back[p].r.soc = INVALID_SOCKET;
  459.       strcpybuff(back[q].url_adr, tmp.url_adr); // address
  460.       back[q].status = -103;  // alive & waiting
  461.       if ((opt->debug>1) && (opt->log!=NULL)) {
  462.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully preserved #%d (%s)"LF, 
  463.           back[q].r.debugid,
  464.           back[q].url_adr); test_flush;
  465.       }
  466.     } else {
  467.       deletehttp(&back[p].r);
  468.       back[p].r.soc = INVALID_SOCKET;
  469.     }
  470.   }
  471. }
  472.  
  473.  
  474. /* attempt to attach a live connection to this slot */
  475. int back_trylive(httrackp* opt,cache_back* cache,lien_back* back, int back_max, int p) {
  476.   if (p>=0 && back[p].status != -103) {     // we never know..
  477.     int i = back_searchlive(opt,back, back_max, back[p].url_adr);   // search slot
  478.     if (i >= 0 && i != p) {
  479.       deletehttp(&back[p].r);               // security check
  480.       back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p
  481.       back_delete(opt,cache,back, i);                 // delete old slot
  482.       back[p].status=100;                   // ready to connect
  483.       return 1;                             // success: will reuse live connection
  484.     }
  485.   }
  486.   return 0;
  487. }
  488.  
  489. /* search for a live position, or, if not possible, try to return a new one */
  490. int back_searchlive(httrackp* opt, lien_back* back, int back_max, char* search_addr) {
  491.   int i;
  492.  
  493.   /* search for a live socket */
  494.   for(i = 0 ; i < back_max ; i++ ) {
  495.     if (back[i].status == -103) {
  496.       if (strfield2(back[i].url_adr, search_addr)) {   /* same location (xxc: check also virtual hosts?) */
  497.         if (time_local() < back[i].ka_time_start + back[i].r.keep_alive_t) {
  498.           return i;
  499.         }
  500.       }
  501.     }
  502.   }
  503.   return -1;
  504. }
  505.   
  506. int back_search(httrackp* opt,cache_back* cache,lien_back* back, int back_max) {
  507.   int i;
  508.  
  509.   /* try to find an empty place */
  510.   for(i = 0 ; i < back_max ; i++ ) {
  511.     if (back[i].status == -1) {
  512.       return i;
  513.     }
  514.   }
  515.  
  516.   /* couldn't find an empty place, try to requisition a keep-alive place */
  517.   for(i = 0 ; i < back_max ; i++ ) {
  518.     if (back[i].status == -103) {
  519.       /* close this place */
  520.       back_delete(opt,cache,back, i);
  521.       return i;
  522.     }
  523.   }
  524.  
  525.   /* oops, can't find a place */
  526.   return -1;
  527. }
  528.  
  529. // effacer entrΘe
  530. int back_delete(httrackp* opt, cache_back* cache, lien_back* back, int p) {
  531.   if (p>=0) {    // on sait jamais..
  532.     // VΘrificateur d'intΘgritΘ
  533.     #if DEBUG_CHECKINT
  534.     _CHECKINT(&back[p],"Appel back_delete")
  535.     #endif
  536. #if HTS_DEBUG_CLOSESOCK
  537.     DEBUG_W("back_delete: #%d\n" _ (int) p);
  538. #endif
  539.     
  540.     // Finalize
  541.     if (!back[p].finalized) {
  542.       if (
  543.         (back[p].status == 0)      // ready
  544.         &&
  545.         (!back[p].testmode)        // not test mode
  546.         &&
  547.         (back[p].r.statuscode>0)   // not internal error
  548.         ) {
  549.         if ((opt->debug>1) && (opt->log!=NULL)) {
  550.           fspc(opt->log,"debug"); fprintf(opt->log,"File '%s%s' -> %s not yet saved in cache - saving now"LF, back[p].url_adr, back[p].url_fil, back[p].url_sav); test_flush;
  551.         }
  552.       }
  553.       back_finalize(opt, cache, back, p);
  554.     }
  555.     back[p].finalized = 0;
  556.     
  557.     // LibΘrer tous les sockets, handles, buffers..
  558.     if (back[p].r.soc!=INVALID_SOCKET) {
  559. #if HTS_DEBUG_CLOSESOCK
  560.       DEBUG_W("back_delete: deletehttp\n");
  561. #endif
  562.       deletehttp(&back[p].r);
  563.       back[p].r.soc=INVALID_SOCKET;
  564.     }
  565.     
  566.     if (back[p].r.adr!=NULL) {  // reste un bloc α dΘsallouer
  567.       freet(back[p].r.adr);
  568.       back[p].r.adr=NULL;
  569.     }
  570.     if (back[p].chunk_adr!=NULL) {  // reste un bloc α dΘsallouer
  571.       freet(back[p].chunk_adr);
  572.       back[p].chunk_adr=NULL;
  573.       back[p].chunk_size=0;
  574.       back[p].chunk_blocksize=0;
  575.       back[p].is_chunk=0;
  576.     }
  577.     // if (back[p].r.is_file) {  // fermer fichier entrΘe
  578.     if (back[p].r.fp!=NULL) {
  579.       fclose(back[p].r.fp);
  580.       back[p].r.fp=NULL;
  581.     }
  582.     // }
  583.  
  584.     // headers
  585.     if (back[p].r.headers != NULL) {
  586.       freet(back[p].r.headers);
  587.       back[p].r.headers = NULL;
  588.     }
  589.  
  590.     /* fichier de sortie */
  591.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  592.       fclose(back[p].r.out);
  593.       back[p].r.out=NULL;
  594.     }
  595.  
  596.     if (back[p].r.is_write) {     // ecriture directe
  597.       /* Θcrire date "remote" */
  598.       if (strnotempty(back[p].url_sav))          // normalement existe si on a un fichier de sortie
  599.       if (strnotempty(back[p].r.lastmodified))   // last-modified existe
  600.       if (fexist(back[p].url_sav))          // ainsi que le fichier
  601.         set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified);
  602.  
  603.       /* executer commande utilisateur aprΦs chargement du fichier */
  604.       //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil);
  605.       back[p].r.is_write=0;
  606.     }
  607.     
  608.     // Tout nettoyer
  609.     memset(&back[p], 0, sizeof(lien_back));  
  610.     back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  611.     
  612.     // Le plus important: libΘrer le champ
  613.     back[p].status=-1;
  614.   }
  615.   return 0;
  616. }
  617.  
  618. /* Space left on backing stack */
  619. int back_stack_available(lien_back* back,int back_max) {
  620.   int p=0,n=0;
  621.   for( ; p < back_max ; p++ )
  622.     if ( back[p].status == -1 )
  623.       n++;
  624.   return n;
  625. }
  626.  
  627. // ajouter un lien en backing
  628. int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) {
  629.   int p=0;
  630.  
  631.   // vΘrifier cohΘrence de adr et fil (non vide!)
  632.   if (strnotempty(adr)==0) {
  633.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  634.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: adr is empty for back_add"LF);
  635.     }
  636.     return -1;    // erreur!
  637.   }
  638.   if (strnotempty(fil)==0) {
  639.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  640.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: fil is empty for back_add"LF);
  641.     }
  642.     return -1;    // erreur!
  643.   }
  644.   // FIN vΘrifier cohΘrence de adr et fil (non vide!)
  645.  
  646.   // stats
  647.   opt->state.back_add_stats++;
  648.  
  649.   // rechercher emplacement
  650.   back_clean(opt, cache, back, back_max);
  651.   if ( ( p = back_search(opt, cache, back, back_max) ) >= 0) {
  652.     back[p].send_too[0]='\0';  // Θventuels paramΦtres supplΘmentaires α transmettre au serveur
  653.  
  654.     // clear r
  655.     if (back[p].r.soc!=INVALID_SOCKET) {  /* we never know */
  656.       deletehttp(&back[p].r);
  657.     }
  658.     memset(&(back[p].r), 0, sizeof(htsblk)); 
  659.     back[p].r.soc=INVALID_SOCKET; 
  660.     back[p].r.location=back[p].location_buffer;
  661.  
  662.     // crΘer entrΘe
  663.     strcpybuff(back[p].url_adr,adr);
  664.     strcpybuff(back[p].url_fil,fil);
  665.     strcpybuff(back[p].url_sav,save);
  666.     back[p].pass2_ptr=pass2_ptr;
  667.     // copier referer si besoin
  668.     strcpybuff(back[p].referer_adr,"");
  669.     strcpybuff(back[p].referer_fil,"");
  670.     if ((referer_adr) && (referer_fil)) {       // existe
  671.       if ((strnotempty(referer_adr)) && (strnotempty(referer_fil))) {   // non vide
  672.         if (referer_adr[0]!='!') {    // non dΘtruit
  673.           if (strcmp(referer_adr,"file://")) {      // PAS file://
  674.             if (strcmp(referer_adr,"primary")) {      // pas referer 1er lien
  675.               strcpybuff(back[p].referer_adr,referer_adr);
  676.               strcpybuff(back[p].referer_fil,referer_fil);
  677.             }
  678.           }
  679.         }
  680.       }
  681.     }
  682.     // sav ne sert α rien pour le moment
  683.     back[p].r.size=0;                   // rien n'a encore ΘtΘ chargΘ
  684.     back[p].r.adr=NULL;                 // pas de bloc de mΘmoire
  685.     back[p].r.is_write=0;               // α priori stockage en mΘmoire
  686.     back[p].maxfile_html=opt->maxfile_html;
  687.     back[p].maxfile_nonhtml=opt->maxfile_nonhtml;
  688.     back[p].testmode=test;              // mode test?
  689.     if (!opt->http10)                 // option "forcer 1.0" dΘsactivΘe
  690.       back[p].http11=1;               // autoriser http/1.1
  691.     back[p].head_request=0;
  692.     if (strcmp(back[p].url_sav,BACK_ADD_TEST)==0)    // HEAD
  693.       back[p].head_request=1;
  694.     else if (strcmp(back[p].url_sav,BACK_ADD_TEST2)==0)    // test en GET
  695.       back[p].head_request=2;       // test en get
  696.  
  697.     /* Stop requested - abort backing */
  698.     if (opt->state.stop) {
  699.       back[p].r.statuscode=-1;        // fatal
  700.       strcpybuff(back[p].r.msg,"mirror stopped by user");
  701.       back[p].status=0;  // terminΘ
  702.       if ((opt->debug>0) && (opt->log!=NULL)) {
  703.         fspc(opt->log,"warning"); fprintf(opt->log,"File not added due to mirror cancel: %s%s"LF,adr,fil); test_flush;
  704.       }            
  705.       return 0;
  706.     }
  707.  
  708.     // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes
  709.     if (cache->cached_tests != NULL) {
  710.       long int ptr = 0;
  711.       if (inthash_read((inthash)cache->cached_tests, concat(adr, fil), (long int*)&ptr)) {    // gotcha
  712.         if (ptr != 0) {
  713.           char* text = (char*) ptr;
  714.           char* lf = strchr(text, '\n');
  715.           int code = 0;
  716.           if (sscanf(text, "%d", &code) == 1) {     // got code
  717.              back[p].r.statuscode=code;
  718.              if (lf != NULL && *lf != '\0') {     // got location ?
  719.                strcpybuff(back[p].r.location, lf + 1);
  720.              }
  721.              return 0;
  722.           }
  723.         }
  724.       }
  725.     }
  726.  
  727.     // tester cache
  728.     if ((strcmp(adr,"file://"))           /* pas fichier */
  729.       && ( (!test) || (cache->type==1) )   /* cache prioritaire, laisser passer en test! */
  730.       && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) {  // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
  731.       //if ((!test) && (strcmp(adr,"file://")) 
  732.       //if ((!test) && (strncmp(adr,"ftp://",6)) && (strcmp(adr,"file://")) 
  733. #if HTS_FAST_CACHE
  734.       long int hash_pos;
  735.       int hash_pos_return=0;
  736. #else
  737.       char* a=NULL;
  738. #endif
  739. #if HTS_FAST_CACHE
  740.       if (cache->hashtable) { 
  741. #else
  742.       if (cache->use) { 
  743. #endif
  744.         char buff[HTS_URLMAXSIZE*4];
  745. #if HTS_FAST_CACHE
  746.         strcpybuff(buff,adr); strcatbuff(buff,fil);
  747.         hash_pos_return=inthash_read((inthash)cache->hashtable,buff,(long int*)&hash_pos);
  748. #else
  749.         buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n");
  750.         a=strstr(cache->use,buff);
  751. #endif
  752.         
  753.         // Ok, notΘ en cache->. mais bien prΘsent dans le cache ou sur disque?
  754. #if HTS_FAST_CACHE
  755.         if (hash_pos_return) {
  756. #else
  757.         if (a) {
  758. #endif
  759.           if (!test) {      // non mode test
  760. #if HTS_FAST_CACHE
  761.             int pos=hash_pos;
  762. #else
  763.             int pos=-1;
  764.             a+=strlen(buff);
  765.             sscanf(a,"%d",&pos);    // lire position
  766. #endif
  767.             if (pos<0) {    // pas de mise en cache data, vΘrifier existence
  768.               if (fsize(fconv(save)) <= 0) {  // fichier existe pas ou est vide!
  769.                 int found=0;
  770.  
  771.                 /* It is possible that the file has been moved due to changes in build structure */
  772.                 {
  773.                   char previous_save[HTS_URLMAXSIZE*2];
  774.                   previous_save[0] = '\0';
  775.                   back[p].r = cache_readex(opt, cache, adr, fil, NULL, back[p].location_buffer, previous_save, 0);
  776.                   if (previous_save[0] != '\0' && fexist(fconv(previous_save))) {
  777.                     rename(fconv(previous_save), fconv(save));
  778.                     if (fexist(fconv(save))) {
  779.                       found = 1;
  780.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  781.                         fspc(opt->log,"debug"); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush;
  782.                       }
  783.                     } else {
  784.                       if ((opt->debug>0) && (opt->log!=NULL)) {
  785.                         fspc(opt->log,"error"); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush;
  786.                       }
  787.                     }
  788.                   }
  789.                 }
  790.                 
  791.                 if (!found) {
  792. #if HTS_FAST_CACHE
  793.                   hash_pos_return=0;
  794. #else
  795.                   a=NULL;    
  796. #endif
  797.                   // dΘvalider car non prΘsent sur disque dans structure originale!!!
  798.                   // sinon, le fichier est ok α priori, mais on renverra un if-modified-since pour
  799.                   // en Ωtre s√r
  800.                   if (opt->norecatch) {              // tester norecatch
  801.                     if (!fexist(fconv(save))) {  // fichier existe pas mais dΘclarΘ: on l'a effacΘ
  802.                       FILE* fp=fopen(fconv(save),"wb");
  803.                       if (fp) fclose(fp);
  804.                       if (opt->log!=NULL) {
  805.                         fspc(opt->log,"warning"); fprintf(opt->log,"File must have been erased by user, ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  806.                       }
  807.                     }
  808.                   }
  809.                 }
  810.               }
  811.             }
  812.           }
  813.         }
  814.         //
  815.       } else
  816. #if HTS_FAST_CACHE
  817.         hash_pos_return=0;
  818. #else
  819.         a=NULL;
  820. #endif
  821.  
  822.       // Existe pas en cache, ou bien pas de cache prΘsent
  823. #if HTS_FAST_CACHE
  824.       if (hash_pos_return) {  // OK existe en cache (et donnΘes aussi)!
  825. #else
  826.       if (a!=NULL) {  // OK existe en cache (et donnΘes aussi)!
  827. #endif
  828.         if (cache->type==1) {   // cache prioritaire (pas de test if-modified..)
  829.                                // dans ce cas on peut Θgalement lire des rΘponses cachΘes comme 404,302...
  830.           // lire dans le cache
  831.           if (!test)
  832.             back[p].r = cache_read(opt,cache,adr,fil,save, back[p].location_buffer);
  833.           else
  834.             back[p].r = cache_read(opt,cache,adr,fil,NULL, back[p].location_buffer);  // charger en tΩte uniquement du cache
  835.  
  836.           /* ensure correct location buffer set */
  837.           back[p].r.location=back[p].location_buffer;
  838.  
  839.           /* Interdiction taille par le wizard? --> dΘtruire */
  840.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  841.             if (!back_checksize(opt,&back[p],0)) {
  842.               back[p].status=0;  // FINI
  843.               back[p].r.statuscode=-1;
  844.               if (!back[p].testmode)
  845.                 strcpybuff(back[p].r.msg,"Cached file skipped (too big)");
  846.               else
  847.                 strcpybuff(back[p].r.msg,"Test: Cached file skipped  (too big)");
  848.               return 0;
  849.             }
  850.           }
  851.  
  852.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  853.             if ((opt->debug>0) && (opt->log!=NULL)) {
  854.               if (!test) {
  855.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately loaded from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  856.               } else {
  857.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately tested from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  858.               }
  859.             }
  860.             back[p].r.notmodified=1;    // fichier non modifiΘ
  861.             back[p].status=0;  // OK prΩt
  862.  
  863.             // finalize transfer
  864.             if (!test) {
  865.               if (back[p].r.statuscode>0) {
  866.                 back_finalize(opt,cache,back,p);
  867.               }
  868.             }
  869.  
  870.             return 0;
  871.           } else {  // erreur
  872.             // effacer r
  873.             memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  874.             // et continuer (chercher le fichier)
  875.           }
  876.           
  877.         } else if (cache->type==2) {    // si en cache, demander de tester If-Modified-Since
  878.           htsblk r;
  879.           cache_header(opt,cache,adr,fil,&r);
  880.  
  881.           /* Interdiction taille par le wizard? */
  882.           {
  883.             LLint save_totalsize=back[p].r.totalsize;
  884.             back[p].r.totalsize=r.totalsize;
  885.             if (!back_checksize(opt,&back[p],1)) {
  886.               r.statuscode = -1;
  887.               //
  888.               back[p].status=0;  // FINI
  889.               deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET;
  890.               if (!back[p].testmode)
  891.                 strcpybuff(back[p].r.msg,"File too big");
  892.               else
  893.                 strcpybuff(back[p].r.msg,"Test: File too big");
  894.               return 0;
  895.             }
  896.             back[p].r.totalsize=save_totalsize;
  897.           }
  898.           
  899.           if (r.statuscode != -1) {
  900.             if (r.statuscode==200) {     // uniquement des 200 (OK)
  901.               if (strnotempty(r.etag)) {  // ETag (RFC2616)
  902.                 /*
  903.                 - If both an entity tag and a Last-Modified value have been
  904.                 provided by the origin server, SHOULD use both validators in
  905.                 cache-conditional requests. This allows both HTTP/1.0 and
  906.                 HTTP/1.1 caches to respond appropriately.
  907.                 */
  908.                 if (strnotempty(r.lastmodified))
  909.                   sprintf(back[p].send_too,"If-None-Match: %s\r\nIf-Modified-Since: %s\r\n",r.etag,r.lastmodified);
  910.                 else
  911.                   sprintf(back[p].send_too,"If-None-Match: %s\r\n",r.etag);
  912.               }
  913.               else if (strnotempty(r.lastmodified))
  914.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",r.lastmodified);
  915.               else if (strnotempty(cache->lastmodified))
  916.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified);
  917.               
  918.               /* this is an update of a file */
  919.               if (strnotempty(back[p].send_too))
  920.                 back[p].is_update=1;
  921.               back[p].r.req.nocompression=1;   /* Do not compress when updating! */
  922.               
  923.             }
  924.           }
  925. #if DEBUGCA
  926.           printf("..is modified test %s\n",back[p].send_too);
  927. #endif
  928.         } 
  929.         // Okay, pas trouvΘ dans le cache
  930.         // Et si le fichier existe sur disque?
  931.         // Pas dans le cache: fichier n'a pas ΘtΘ transfΘrΘ du tout, donc pas sur disque?
  932.       } else {
  933.         if (fexist(save)) {    // fichier existe? aghl!
  934.           LLint sz=fsize(save);
  935.           // Bon, lα il est possible que le fichier ait ΘtΘ partiellement transfΘrΘ
  936.           // (s'il l'avait ΘtΘ en totalitΘ il aurait ΘtΘ inscrit dans le cache ET existerait sur disque)
  937.           // PAS de If-Modified-Since, on a pas connaissance des donnΘes α la date du cache
  938.           // On demande juste les donnΘes restantes si le date est valide (206), tout sinon (200)
  939.           if ((ishtml(save) != 1) && (ishtml(back[p].url_fil)!=1)) {   // NON HTML (liens changΘs!!)
  940.             if (sz>0) {    // Fichier non vide? (question bΩte, sinon on transfert tout!)
  941.               char lastmodified[256];
  942.               get_filetime_rfc822(save, lastmodified);
  943.               if (strnotempty(lastmodified)) {     /* pas de If-.. possible */
  944. #if DEBUGCA
  945.                 printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz);
  946. #endif
  947.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  948.                   fspc(opt->log,"debug"); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  949.                 }
  950.                 
  951.                 /* impossible - don't have etag or date
  952.                 if (strnotempty(back[p].r.etag)) {  // ETag (RFC2616)
  953.                 sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag);
  954.                 back[p].http11=1;    // En tΩte 1.1
  955.                 } else if (strnotempty(back[p].r.lastmodified)) {
  956.                 sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified);
  957.                 back[p].http11=1;    // En tΩte 1.1
  958.                 } else 
  959.                 */
  960.                 if (strlen(lastmodified)) {
  961.                   sprintf(back[p].send_too,
  962.                     "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n"
  963.                     , lastmodified, (LLint)sz);
  964.                   back[p].http11=1;    // En tΩte 1.1
  965.                   back[p].range_req_size=sz;
  966.                   back[p].r.req.range_used=1;
  967.                   back[p].r.req.nocompression=1;
  968.                 } else {
  969.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  970.                     fspc(opt->log,"warning"); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  971.                   }
  972.                 }
  973.                 
  974.               } else { 
  975.                 if ((opt->debug>0) && (opt->errlog!=NULL)) {
  976.                   fspc(opt->errlog,"warning");
  977.                   /*
  978.                   if (opt->http10)
  979.                   fprintf(opt->errlog,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil);
  980.                   else
  981.                   */
  982.                   fprintf(opt->errlog,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  983.                   test_flush;
  984.                 }
  985.                 /* Sinon requΩte normale... */
  986.                 back[p].http11=0;
  987.               }
  988.             } else if (opt->norecatch) {              // tester norecatch
  989.               filenote(save,NULL);       // ne pas purger tout de mΩme
  990.               back[p].status=0;  // OK prΩt
  991.               back[p].r.statuscode=-1;  // erreur
  992.               strcpybuff(back[p].r.msg,"Null-size file not recaught");
  993.               return 0;
  994.             }
  995.           } else {
  996.             if ((opt->debug>0) && (opt->errlog!=NULL)) {
  997.               fspc(opt->errlog,"warning");
  998.               fprintf(opt->errlog,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  999.               test_flush;
  1000.             }
  1001.             /* Sinon requΩte normale... */
  1002.             back[p].http11=0;
  1003.           }
  1004.         }
  1005.       }
  1006.     }
  1007.  
  1008.  
  1009.     {
  1010.       ///htsblk r;   non directement dans la structure-rΘponse!
  1011.       T_SOC soc;
  1012.       
  1013.       // ouvrir liaison, envoyer requΦte
  1014.       // ne pas traiter ou recevoir l'en tΩte immΘdiatement
  1015.       memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  1016.       // recopier proxy
  1017.       memcpy(&(back[p].r.req.proxy), &opt->proxy, sizeof(opt->proxy));
  1018.       // et user-agent
  1019.       strcpybuff(back[p].r.req.user_agent,opt->user_agent);
  1020.       strcpybuff(back[p].r.req.referer,opt->referer);
  1021.       strcpybuff(back[p].r.req.from,opt->from);
  1022.       strcpybuff(back[p].r.req.lang_iso,opt->lang_iso);
  1023.       back[p].r.req.user_agent_send=opt->user_agent_send;
  1024.       // et http11
  1025.       back[p].r.req.http11=back[p].http11;
  1026.       back[p].r.req.nocompression=opt->nocompression;
  1027.       back[p].r.req.nokeepalive=opt->nokeepalive;
  1028.  
  1029.       // mode ftp, court-circuit!
  1030.       if (strfield(back[p].url_adr,"ftp://")) {
  1031.         if (back[p].testmode) {
  1032.           if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1033.             fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: forbidden test with ftp link for back_add"LF);
  1034.           }
  1035.           return -1;    // erreur pas de test permis
  1036.         }
  1037.         if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gΘrΘe en thread
  1038.           back[p].status=1000;   // connexion ftp
  1039. #if USE_BEGINTHREAD
  1040.           launch_ftp(&(back[p]));
  1041. #else
  1042.           {
  1043.             char nid[32];
  1044.             sprintf(nid,"htsftp%d-in_progress.lock",p);
  1045.             strcpybuff(back[p].location_buffer,fconcat(opt->path_log,nid));
  1046.           }
  1047.           launch_ftp(&(back[p]),back[p].location_buffer,opt->exec);
  1048. #endif
  1049.           return 0;
  1050.         }
  1051.       }
  1052. #if HTS_USEOPENSSL
  1053.       else if (SSL_is_available && strfield(back[p].url_adr,"https://")) {        // let's rock
  1054.         back[p].r.ssl = 1;
  1055.         // back[p].r.ssl_soc = NULL;
  1056.         back[p].r.ssl_con = NULL;
  1057.       }
  1058. #endif
  1059.       
  1060.       if (!back_trylive(opt, cache, back, back_max, p)) {
  1061. #if HTS_XGETHOST
  1062. #if HDEBUG
  1063.         printf("back_solve..\n");
  1064. #endif
  1065.         back[p].status=101;    // tentative de rΘsolution du nom de host
  1066.         soc=INVALID_SOCKET;    // pas encore ouverte
  1067.         back_solve(&back[p]);  // prΘparer
  1068.         if (host_wait(&back[p])) {  // prΩt, par ex fichier ou dispo dans dns
  1069. #if HDEBUG
  1070.           printf("ok, dns cache ready..\n");
  1071. #endif
  1072.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1073.           if (soc==INVALID_SOCKET) {
  1074.             back[p].status=0;  // fini, erreur
  1075.           }
  1076.         }
  1077.         //
  1078. #else
  1079.         //
  1080. #if CNXDEBUG
  1081.         printf("XFopen..\n");
  1082. #endif
  1083.         
  1084.         if (strnotempty(back[p].send_too))    // envoyer un if-modified-since
  1085. #if HTS_XCONN
  1086.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1087. #else
  1088.         soc=http_xfopen(0,0,1,back[p].send_too,adr,fil,&(back[p].r));
  1089. #endif
  1090.         else
  1091. #if HTS_XCONN
  1092.           soc=http_xfopen(test,0,0,NULL,adr,fil,&(back[p].r));
  1093. #else
  1094.         soc=http_xfopen(test,0,1,NULL,adr,fil,&(back[p].r));
  1095. #endif
  1096. #endif
  1097.       } else {
  1098.         soc = back[p].r.soc;
  1099.  
  1100.         if ((opt->debug>1) && (opt->log!=NULL)) {
  1101.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully linked #%d (for %s%s)"LF, 
  1102.             back[p].r.debugid,
  1103.             back[p].url_adr, back[p].url_fil); test_flush;
  1104.         }
  1105.       }
  1106.       
  1107.       if (opt->timeout>0) {    // gestion du opt->timeout
  1108.         back[p].timeout=opt->timeout;
  1109.         back[p].timeout_refresh=time_local();
  1110.       } else {
  1111.         back[p].timeout=-1;    // pas de gestion (default)
  1112.       }
  1113.       
  1114.       if (opt->rateout>0) {    // gestion d'un taux minimum de transfert tolΘrΘ
  1115.         back[p].rateout=opt->rateout;
  1116.         back[p].rateout_time=time_local();
  1117.       } else {
  1118.         back[p].rateout=-1;    // pas de gestion (default)
  1119.       }
  1120.  
  1121.       // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas o∙ cela est
  1122.       // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
  1123.       //if ((back[p].statuscode!=200) || (soc<0)) { // ERREUR HTTP/autre
  1124.  
  1125. #if CNXDEBUG
  1126. printf("Xfopen ok, poll..\n");
  1127. #endif
  1128.  
  1129. #if HTS_XGETHOST
  1130.     if (soc!=INVALID_SOCKET)
  1131.       if (back[p].status==101) {  // pas d'erreur
  1132.         if (!back[p].r.is_file)
  1133.           back[p].status=100;   // connexion en cours
  1134.         else
  1135.           back[p].status=1;     // fichier
  1136.       }
  1137.  
  1138. #else
  1139.       if (soc==INVALID_SOCKET) { // erreur socket
  1140.         back[p].status=0;    // FINI
  1141.         //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
  1142.         back[p].r.soc=INVALID_SOCKET;
  1143.       } else {
  1144.         if (!back[p].r.is_file)
  1145. #if HTS_XCONN
  1146.           back[p].status=100;   // connexion en cours
  1147. #else
  1148.           back[p].status=99;    // chargement en tΩte en cours
  1149. #endif
  1150.         else
  1151.           back[p].status=1;     // chargement fichier
  1152. #if BDEBUG==1
  1153.         printf("..loading header\n");
  1154. #endif
  1155.       }
  1156. #endif
  1157.       
  1158.     }
  1159.  
  1160.  
  1161.     // note: si il y a erreur (404,etc) status=2 (terminΘ/Θchec) mais
  1162.     // le lien est considΘrΘ comme traitΘ
  1163.     //if (back[p].soc<0)  // erreur
  1164.     //  return -1;
  1165.  
  1166.     return 0;
  1167.   } else {
  1168.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1169.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: no space left in stack for back_add"LF);
  1170.     }
  1171.     return -1;    // plus de place
  1172.   }
  1173. }
  1174.  
  1175.  
  1176.  
  1177. #if HTS_XGETHOST
  1178. #if USE_BEGINTHREAD
  1179. // lancement multithread du robot
  1180. PTHREAD_TYPE Hostlookup(void* iadr_p) {
  1181.   char iadr[256];
  1182.   t_dnscache* cache=_hts_cache();  // adresse du cache
  1183.   t_hostent* hp;
  1184.   int error_found=0;
  1185.  
  1186.   // recopier (aprΦs id:pass)
  1187. #if DEBUGDNS 
  1188.   printf("resolv in background: %s\n",jump_identification(iadr_p));
  1189. #endif
  1190.   strcpybuff(iadr,jump_identification(iadr_p));
  1191.   // couper Θventuel :
  1192.   {
  1193.     char *a;
  1194.     if ( (a=jump_toport(iadr)) )
  1195.       *a='\0';          // get rid of it
  1196.   }
  1197.   freet(iadr_p);
  1198.  
  1199.   // attendre que le cache dns soit prΩt
  1200.   while(_hts_lockdns(-1));  // attendre libΘration
  1201.   _hts_lockdns(1);          // locker
  1202.   while(cache->n) {
  1203.     if (strcmp(cache->iadr,iadr)==0) {
  1204.       error_found=1;
  1205.     }
  1206.     cache=cache->n;    // calculer queue
  1207.   }
  1208.   if (strcmp(cache->iadr,iadr)==0) {
  1209.     error_found=1;
  1210.   }
  1211.  
  1212.   if (!error_found) {
  1213.     // en gros copie de hts_gethostbyname sans le return
  1214.     cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache));
  1215.     if (cache->n!=NULL) {
  1216.       t_fullhostent fullhostent_buffer;
  1217.       strcpybuff(cache->n->iadr,iadr);
  1218.       cache->n->host_length=0;        /* pour le moment rien */
  1219.       cache->n->n=NULL;
  1220.       _hts_lockdns(0);          // dΘlocker
  1221.       
  1222.       /* resolve */
  1223. #if DEBUGDNS 
  1224.       printf("gethostbyname() in progress for %s\n",iadr);
  1225. #endif
  1226.       cache->n->host_length=-1;
  1227.       memset(cache->n->host_addr, 0, sizeof(cache->n->host_addr));
  1228.       hp=vxgethostbyname(iadr, &fullhostent_buffer);
  1229.       if (hp!=NULL) {
  1230.         memcpy(cache->n->host_addr, hp->h_addr, hp->h_length);
  1231.         cache->n->host_length = hp->h_length;
  1232.       }
  1233.     } else 
  1234.     _hts_lockdns(0);          // dΘlocker
  1235.   } else {
  1236. #if DEBUGDNS 
  1237.     printf("aborting resolv for %s (found)\n",iadr);
  1238. #endif
  1239.     _hts_lockdns(0);          // dΘlocker
  1240.   }
  1241.   // fin de copie de hts_gethostbyname
  1242.  
  1243. #if DEBUGDNS 
  1244.   printf("quitting resolv for %s (result: %d)\n",iadr,(cache->n!=NULL)?cache->n->host_length:(-999));
  1245. #endif
  1246.  
  1247.   return PTHREAD_RETURN;     /* _endthread implied  */
  1248. }
  1249. #endif
  1250.  
  1251. // attendre que le host (ou celui du proxy) ait ΘtΘ rΘsolu
  1252. // si c'est un fichier, la rΘsolution est immΘdiate
  1253. // idem pour ftp://
  1254. void back_solve(lien_back* back) {
  1255.   if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) {
  1256.   //## if (back->url_adr[0]!=lOCAL_CHAR) {  // qq chose α prΘparer
  1257.     char* a;
  1258.     if (!(back->r.req.proxy.active))
  1259.       a=back->url_adr;
  1260.     else
  1261.       a=back->r.req.proxy.name;
  1262.     a = jump_protocol(a);
  1263.     if (!hts_dnstest(a)) {   // non encore testΘ!..
  1264.       // inscire en thread
  1265. #if HTS_WIN
  1266.       // Windows
  1267. #if USE_BEGINTHREAD
  1268.       {
  1269.         char* p = calloct(strlen(a)+2,1);
  1270.         if (p) {
  1271.           strcpybuff(p,a);
  1272.           _beginthread( Hostlookup , 0, p );
  1273.         }
  1274.       }
  1275. #else
  1276.       /*t_hostent* h=*/
  1277.       /*hts_gethostbyname(a);*/  // calcul
  1278. #endif
  1279. #else
  1280. #if USE_BEGINTHREAD
  1281.         char* p = calloct(strlen(a)+2,1);
  1282.         if (p) {
  1283.           strcpybuff(p,a);
  1284.           _beginthread( Hostlookup , 0, p );
  1285.         }
  1286. #else
  1287.       // Sous Unix, le gethostbyname() est bloquant..
  1288.       /*t_hostent* h=*/
  1289.       /*hts_gethostbyname(a);*/  // calcul
  1290. #endif
  1291. #endif
  1292.     }
  1293.   }
  1294. }
  1295.  
  1296. // dΘtermine si le host a pu Ωtre rΘsolu
  1297. int host_wait(lien_back* back) {
  1298.   if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) {
  1299.   //## if (back->url_adr[0]!=lOCAL_CHAR) {
  1300.     if (!(back->r.req.proxy.active)) {
  1301.       return (hts_dnstest(back->url_adr));
  1302.     } else {
  1303.       return (hts_dnstest(back->r.req.proxy.name));      
  1304.     }
  1305.   } else return 1;    // prΩt, fichier local
  1306. }
  1307. #endif
  1308.  
  1309.  
  1310. // Θlimine les fichiers non html en backing (anticipation)
  1311. // cleanup non-html files in backing to save backing space
  1312. // and allow faster "save in cache" operation
  1313. // also cleanup keep-alive sockets and ensure that not too many sockets are being opened
  1314. void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) {
  1315. #if HTS_ANALYSTE
  1316.   int oneMore = ( (_hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (_hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0;  // testing links
  1317. #endif
  1318.   int i;
  1319.   for(i=0;i<back_max;i++) {
  1320.     if (back[i].status == 0) {                                   // ready
  1321.       /* Check autoclean */
  1322.       if (!back[i].testmode) {                                   // not test mode
  1323.         if (strnotempty(back[i].url_sav)) {                      // filename exists
  1324.           if (back[i].r.statuscode==200) {                   // HTTP "OK"
  1325.             if (back[i].r.size>0) {                              // size>0
  1326.               if (back[i].r.is_write                                // not in memory (on disk, ready)
  1327.                 && !is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)        // not HTML/hypertext
  1328.                 && !may_be_hypertext_mime(back[i].r.contenttype, back[i].url_fil)    // may NOT be parseable mime type
  1329.                 ) {
  1330.                 if (back[i].pass2_ptr) {
  1331.                   // finalize
  1332.                   // // back_finalize(opt,cache,back,i);
  1333.                   // stats
  1334.                   //HTS_STAT.stat_bytes+=back[i].r.size;
  1335.                   //HTS_STAT.stat_files++;
  1336.                   //if ( (!back[i].r.notmodified) && (opt->is_update) ) { 
  1337.                   //  HTS_STAT.stat_updated_files++;       // page modifiΘe
  1338.                   //}
  1339.                   //xxxcache_mayadd(opt,cache,&back[i].r,back[i].url_adr,back[i].url_fil,back[i].url_sav);
  1340.                   usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil);
  1341.                   *back[i].pass2_ptr=-1;  // Done!
  1342.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  1343.                     fspc(opt->log,"info"); fprintf(opt->log,"File successfully written in background: %s"LF,back[i].url_sav); test_flush;
  1344.                   }
  1345.                   back_maydelete(opt,cache,back,i);    // May delete backing entry
  1346.                 }
  1347.               } else {
  1348.                 if (!back[i].finalized) {
  1349.                   if (1) {
  1350.                     /* Ensure deleted or recycled socket */
  1351.                     /* BUT DO NOT YET WIPE back[i].r.adr */
  1352.                     if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1353.                       fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s validated (cached, left in memory)"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1354.                     }
  1355.                     back_maydeletehttp(opt, cache, back, back_max, i);
  1356.                   } else {
  1357.                   /*
  1358.                   NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO)
  1359.                     */
  1360.                     /* Lock the entry but do not keep the html data in memory (in cache) */
  1361.                     if (opt->cache) {
  1362.                       htsblk r;
  1363.                       
  1364.                       /* Ensure deleted or recycled socket */
  1365.                       back_maydeletehttp(opt, cache, back, back_max, i);
  1366.                       assertf(back[i].r.soc == INVALID_SOCKET);
  1367.                       
  1368.                       /* Check header */
  1369.                       cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r);
  1370.                       if (r.statuscode == 200) {
  1371.                         if (back[i].r.soc == INVALID_SOCKET) {
  1372.                           /* Delete buffer and sockets */
  1373.                           deleteaddr(&back[i].r);
  1374.                           deletehttp(&back[i].r);
  1375.                           if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1376.                             fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s temporarily left in cache to spare memory"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1377.                           }
  1378.                         }
  1379.                       } else {
  1380.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  1381.                           fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected html cache lookup error during back clean"LF); test_flush;
  1382.                         }            
  1383.                       }
  1384.                       // xxc xxc
  1385.                     }
  1386.                   }
  1387.                 }
  1388.               }
  1389.             }
  1390.           }
  1391.         }
  1392.       }
  1393.     } else if (back[i].status == -103) {                         // waiting (keep-alive)
  1394.       if (
  1395.         ! back[i].r.keep_alive
  1396.         || back[i].r.soc == INVALID_SOCKET
  1397.         || back[i].r.keep_alive_max < 1
  1398.         || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t
  1399.         ) {
  1400.         if ((opt->debug>0) && (opt->log!=NULL)) {
  1401.             fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): live socket closed #%d (%s)"LF, 
  1402.             back[i].r.debugid,
  1403.             back[i].url_adr);
  1404.             test_flush;
  1405.         }
  1406.         back_delete(opt,cache,back, i);    // delete backing entry
  1407.       }
  1408.     }
  1409.   }
  1410.   /* switch connections to live ones */
  1411.   for(i=0;i<back_max;i++) {
  1412.     if (back[i].status == 0) {                                   // ready
  1413.       if (back[i].r.soc != INVALID_SOCKET) {
  1414.         back_maydeletehttp(opt,cache,back, back_max, i);
  1415.       }
  1416.       
  1417.     }
  1418.   }
  1419.   /* delete sockets if too many keep-alive'd sockets in background */
  1420.   if (opt->maxsoc > 0) {
  1421.     int max = opt->maxsoc + oneMore;
  1422.     int curr = back_nsoc_overall(back, back_max);
  1423.     if (curr > max) {
  1424.       if ((opt->debug>1) && (opt->log!=NULL)) {
  1425.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): deleting #%d sockets"LF, 
  1426.           curr - max); test_flush;
  1427.       }
  1428.     }
  1429.     for(i = 0 ; i < back_max && curr > max ; i++) {
  1430.       if (back[i].status == -103) {
  1431.         back_delete(opt,cache,back, i);    // delete backing entry
  1432.         curr--;
  1433.       }
  1434.     }
  1435.   }
  1436. }
  1437.  
  1438.  
  1439. // attente (gestion des buffers des sockets)
  1440. void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TStamp stat_timestart) {
  1441.   unsigned int i_mod;
  1442.   T_SOC nfds=INVALID_SOCKET;
  1443.   fd_set fds,fds_c,fds_e;     // fds pour lecture, connect (write), et erreur
  1444.   int nsockets;     // nbre sockets
  1445.   LLint max_read_bytes;  // max bytes read per sockets
  1446.   struct timeval tv;
  1447.   int do_wait=0;
  1448.   int gestion_timeout=0;
  1449.   int busy_recv=0;     // pas de donnΘes pour le moment   
  1450.   int busy_state=0;    // pas de connexions
  1451.   int max_loop;  // nombre de boucles max α parcourir..
  1452. #if HTS_ANALYSTE
  1453.   int max_loop_chk=0;
  1454. #endif
  1455.   unsigned int mod_random = (unsigned int) ( time_local() + HTS_STAT.HTS_TOTAL_RECV );
  1456.  
  1457.   // max. number of loops
  1458.   max_loop=8;
  1459.  
  1460. #if 1
  1461.   // Cleanup the stack to save space!
  1462.   back_clean(opt,cache,back,back_max);
  1463. #endif
  1464.  
  1465.   // recevoir tant qu'il y a des donnΘes (avec un maximum de max_loop boucles)
  1466.   do_wait=0;
  1467.   gestion_timeout=0;
  1468.   do {
  1469.     int max_c;
  1470.     busy_state=busy_recv=0;
  1471.  
  1472. #if 0
  1473.     check_rate(stat_timestart,opt->maxrate);    // vΘrifier taux de transfert
  1474. #endif
  1475.     // inscrire les sockets actuelles, et rechercher l'ID la plus ΘlevΘe
  1476.     FD_ZERO(&fds);
  1477.     FD_ZERO(&fds_c);
  1478.     FD_ZERO(&fds_e);
  1479.     nsockets=0;
  1480.     max_read_bytes=TAILLE_BUFFER;     // maximum bytes that can be read
  1481.     nfds=INVALID_SOCKET;
  1482.  
  1483.     max_c=1;
  1484.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  1485.     // for(i=0;i<back_max;i++) {
  1486.         unsigned int i = ( i_mod + mod_random ) % ( back_max );
  1487.  
  1488.       // en cas de gestion du connect prΘemptif
  1489. #if HTS_XCONN
  1490.       if (back[i].status==100) {      // connexion
  1491.         do_wait=1;
  1492.  
  1493.         // noter socket write
  1494.         FD_SET(back[i].r.soc,&fds_c);
  1495.         
  1496.         // noter socket erreur
  1497.         FD_SET(back[i].r.soc,&fds_e);
  1498.  
  1499.         // calculer max
  1500.         if (max_c) {
  1501.           max_c=0;
  1502.           nfds=back[i].r.soc;
  1503.         } else if (back[i].r.soc>nfds) {
  1504.           // ID socket la plus ΘlevΘe
  1505.           nfds=back[i].r.soc;
  1506.         }
  1507.         
  1508.       } else
  1509. #endif
  1510. #if HTS_XGETHOST
  1511.       if (back[i].status==101) {      // attente
  1512.         // rien α faire..
  1513.       } else
  1514. #endif
  1515.       // poll pour la lecture sur les sockets
  1516.       if ((back[i].status>0) && (back[i].status<100)) {  // en rΘception http
  1517.             
  1518. #if BDEBUG==1
  1519.         //printf("....socket in progress: %d\n",back[i].r.soc);
  1520. #endif
  1521.         // non local et non ftp
  1522.         if (!back[i].r.is_file) {
  1523.         //## if (back[i].url_adr[0]!=lOCAL_CHAR) {
  1524.           
  1525.           // vΘrification de sΘcuritΘ
  1526.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  1527.             do_wait=1;
  1528.             
  1529.             // noter socket read
  1530.             FD_SET(back[i].r.soc,&fds);
  1531.             
  1532.             // noter socket error
  1533.             FD_SET(back[i].r.soc,&fds_e);
  1534.             
  1535.             // incrΘmenter nombre de sockets
  1536.             nsockets++;
  1537.  
  1538.             // calculer max
  1539.             if (max_c) {
  1540.               max_c=0;
  1541.               nfds=back[i].r.soc;
  1542.             } else if (back[i].r.soc>nfds) {
  1543.               // ID socket la plus ΘlevΘe
  1544.               nfds=back[i].r.soc;
  1545.             }
  1546.           } else {
  1547.             back[i].r.statuscode=-4;
  1548.             if (back[i].status==100)
  1549.               strcpybuff(back[i].r.msg,"Connect Error");
  1550.             else
  1551.               strcpybuff(back[i].r.msg,"Receive Error");
  1552.             back[i].status=0;  // terminΘ
  1553.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1554.               fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected socket error during pre-loop"LF); test_flush;
  1555.             }            
  1556.           }
  1557. #if WIDE_DEBUG
  1558.           else {
  1559.             DEBUG_W("PANIC!!! Socket is invalid in a poll test!\n");
  1560.           }
  1561. #endif
  1562.           
  1563.         }
  1564.         
  1565.       }
  1566.     }    
  1567.     nfds++;
  1568.     
  1569.     if (do_wait) {  // attendre
  1570.       // temps d'attente max: 2.5 seconde
  1571.       tv.tv_sec=HTS_SOCK_SEC;
  1572.       tv.tv_usec=HTS_SOCK_MS;
  1573.       
  1574. #if BDEBUG==1
  1575.       printf("..select\n");
  1576. #endif
  1577.       
  1578.       // poller les sockets-attention au noyau sous Unix..
  1579. #if HTS_WIDE_DEBUG    
  1580.       DEBUG_W("select\n");
  1581. #endif
  1582.       select(nfds,&fds,&fds_c,&fds_e,&tv);
  1583. #if HTS_WIDE_DEBUG    
  1584.       DEBUG_W("select done\n");
  1585. #endif      
  1586.     }
  1587.     
  1588.     // maximum data which can be received for a socket, if limited
  1589.     if (nsockets) {
  1590.       if (opt->maxrate>0) {
  1591.         max_read_bytes = ( check_downloadable_bytes(opt->maxrate) / nsockets );
  1592.         if (max_read_bytes > TAILLE_BUFFER) {
  1593.           /* limit size */
  1594.           max_read_bytes = TAILLE_BUFFER;
  1595.         } else if (max_read_bytes < TAILLE_BUFFER) {
  1596.           /* a small pause */
  1597.           Sleep(10);
  1598.         }
  1599.       }
  1600.     }
  1601.     if (!max_read_bytes)
  1602.       busy_recv=0;
  1603.     
  1604.     // recevoir les donnΘes arrivΘes
  1605.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  1606.     // for(i=0;i<back_max;i++) {
  1607.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  1608.       if (back[i].status>0) {
  1609.         if (!back[i].r.is_file) {  // not file..
  1610.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  1611.             int err=FD_ISSET(back[i].r.soc,&fds_e);
  1612.             if (err) {
  1613.               if (back[i].r.soc!=INVALID_SOCKET) {
  1614. #if HTS_DEBUG_CLOSESOCK
  1615.                 DEBUG_W("back_wait: deletehttp\n");
  1616. #endif
  1617.                 deletehttp(&back[i].r);
  1618.               }
  1619.               back[i].r.soc=INVALID_SOCKET;
  1620.               back[i].r.statuscode=-4;
  1621.               if (back[i].status==100)
  1622.                 strcpybuff(back[i].r.msg,"Connect Error");
  1623.               else
  1624.                 strcpybuff(back[i].r.msg,"Receive Error");
  1625.               if (back[i].status == -103) {     /* Keep-alive socket */
  1626.                 back_delete(opt,cache,back, i);
  1627.               } else {
  1628.                 back[i].status=0;  // terminΘ
  1629.               }
  1630.             }
  1631.           }
  1632.         }
  1633.       }
  1634.       
  1635.       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
  1636.       if (back[i].status==100) {   // attendre connect
  1637.         int dispo=0;
  1638.         // vΘrifier l'existance de timeout-check
  1639.         if (!gestion_timeout)
  1640.           if (back[i].timeout>0)
  1641.             gestion_timeout=1;
  1642.           
  1643.           // connectΘ?
  1644.           dispo=FD_ISSET(back[i].r.soc,&fds_c);
  1645.           if (dispo) {    // ok connected!!
  1646.             busy_state=1;
  1647.             
  1648. #if HTS_USEOPENSSL
  1649.             /* SSL mode */
  1650.             if (SSL_is_available && back[i].r.ssl) {
  1651.               // handshake not yet launched
  1652.               if (!back[i].r.ssl_con) {
  1653.                 SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
  1654.                 // new session
  1655.                 back[i].r.ssl_con = SSL_new(openssl_ctx);
  1656.                 if (back[i].r.ssl_con) {
  1657.                   SSL_clear(back[i].r.ssl_con);
  1658.                   if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) {
  1659.                     SSL_set_connect_state(back[i].r.ssl_con);
  1660.                     back[i].status = 102;         /* handshake wait */
  1661.                   } else
  1662.                     back[i].r.statuscode=-6;
  1663.                 } else
  1664.                   back[i].r.statuscode=-6;
  1665.               }
  1666.               /* Error */
  1667.               if (back[i].r.statuscode == -6) {
  1668.                 strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
  1669.                 deletehttp(&back[i].r);
  1670.                 back[i].r.soc=INVALID_SOCKET;
  1671.                 back[i].r.statuscode=-5;
  1672.                 back[i].status=0;
  1673.               }
  1674.             }
  1675.             
  1676. #endif
  1677.  
  1678. #if BDEBUG==1
  1679.           printf("..connect ok on socket %d\n",back[i].r.soc);
  1680. #endif
  1681.           
  1682.           if ((back[i].r.soc != INVALID_SOCKET) && (back[i].status==100)) {
  1683.             /* limit nb. connections/seconds to avoid server overload */
  1684.             if (opt->maxconn>0) {
  1685.               Sleep(1000/opt->maxconn);
  1686.             }
  1687.             
  1688.             back[i].ka_time_start=time_local();
  1689.             if (back[i].timeout>0) {    // refresh timeout si besoin est
  1690.               back[i].timeout_refresh=back[i].ka_time_start;
  1691.             }
  1692.             if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  1693.               back[i].rateout_time=back[i].ka_time_start;
  1694.             }
  1695.             // envoyer header
  1696.             //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
  1697.             HTS_STAT.stat_nrequests++;
  1698.             if (!back[i].head_request)
  1699.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1700.             else if (back[i].head_request==2)  // test en GET!
  1701.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1702.             else        // test!
  1703.               http_sendhead(opt->cookie,1,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1704.             back[i].status=99;  // attendre en tΩte maintenant
  1705.           }
  1706.         }
  1707.         
  1708.         // attente gethostbyname
  1709.       }
  1710. #if HTS_USEOPENSSL
  1711.       else if (SSL_is_available && back[i].status==102) {   // wait for SSL handshake
  1712.         /* SSL mode */
  1713.         if (back[i].r.ssl) {
  1714.           int conn_code;
  1715.           if ((conn_code = SSL_connect(back[i].r.ssl_con)) <= 0) {
  1716.             /* non blocking I/O, will retry */
  1717.             int err_code = SSL_get_error(back[i].r.ssl_con, conn_code);
  1718.             if (
  1719.               (err_code != SSL_ERROR_WANT_READ)
  1720.               &&
  1721.               (err_code != SSL_ERROR_WANT_WRITE)
  1722.               ) {
  1723.               char tmp[256];
  1724.               tmp[0]='\0';
  1725.               ERR_error_string(err_code, tmp);
  1726.               back[i].r.msg[0]='\0';
  1727.               strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2);
  1728.               if (!strnotempty(back[i].r.msg)) {
  1729.                 sprintf(back[i].r.msg, "SSL/TLS error %d", err_code);
  1730.               }
  1731.               deletehttp(&back[i].r);
  1732.               back[i].r.soc=INVALID_SOCKET;
  1733.               back[i].r.statuscode=-5;
  1734.               back[i].status=0;
  1735.             }
  1736.           } else {        /* got it! */
  1737.             back[i].status=100;       // back to waitconnect
  1738.           }
  1739.         } else {
  1740.           strcpybuff(back[i].r.msg, "unexpected SSL/TLS error");
  1741.           deletehttp(&back[i].r);
  1742.           back[i].r.soc=INVALID_SOCKET;
  1743.           back[i].r.statuscode=-5;
  1744.           back[i].status=0;
  1745.         }
  1746.         
  1747.       }
  1748. #endif
  1749. #if HTS_XGETHOST
  1750.       else if (back[i].status==101) {  // attendre gethostbyname
  1751. #if DEBUGDNS 
  1752.         //printf("status 101 for %s\n",back[i].url_adr);
  1753. #endif
  1754.  
  1755.         if (!gestion_timeout)
  1756.           if (back[i].timeout>0)
  1757.             gestion_timeout=1;
  1758.  
  1759.         if (host_wait(&back[i])) {    // prΩt
  1760.           back[i].status=100;        // attente connexion
  1761.           if (back[i].timeout>0) {    // refresh timeout si besoin est
  1762.             back[i].timeout_refresh=time_local();
  1763.           }
  1764.           if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  1765.             back[i].rateout_time=time_local();
  1766.           }
  1767.  
  1768.           back[i].r.soc=http_xfopen(0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r));
  1769.           if (back[i].r.soc==INVALID_SOCKET) {
  1770.             back[i].status=0;  // fini, erreur
  1771.             if (back[i].r.soc!=INVALID_SOCKET) {
  1772. #if HTS_DEBUG_CLOSESOCK
  1773.               DEBUG_W("back_wait(2): deletehttp\n");
  1774. #endif
  1775.               deletehttp(&back[i].r);
  1776.             }
  1777.             back[i].r.soc=INVALID_SOCKET;
  1778.             back[i].r.statuscode=-5;
  1779.             if (strnotempty(back[i].r.msg)==0) 
  1780.               strcpybuff(back[i].r.msg,"Unable to resolve host name");
  1781.           }
  1782.         }
  1783.         
  1784.  
  1785.       // ---- FLAG READ MIS A UN?: POUR LA RECEPTION
  1786.       }
  1787. #endif
  1788. #if USE_BEGINTHREAD
  1789.       // ..rien α faire, c'est magic les threads
  1790. #else
  1791.       else if (back[i].status==1000) {  // en rΘception ftp
  1792.         if (!fexist(back[i].location_buffer)) {    // terminΘ
  1793.           FILE* fp;
  1794.           fp=fopen(fconcat(back[i].location_buffer,".ok"),"rb");
  1795.           if (fp) {
  1796.             int j=0;
  1797.             fscanf(fp,"%d ",&(back[i].r.statuscode));
  1798.             while(!feof(fp)) {
  1799.               int c = fgetc(fp);
  1800.               if (c!=EOF)
  1801.                 back[i].r.msg[j++]=c;
  1802.             }
  1803.             back[i].r.msg[j++]='\0';
  1804.             fclose(fp);
  1805.             remove(fconcat(back[i].location_buffer,".ok"));
  1806.             strcpybuff(fconcat(back[i].location_buffer,".ok"),"");
  1807.           } else {
  1808.             strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok");
  1809.             back[i].r.statuscode=-1;
  1810.           }
  1811.           back[i].status=0;
  1812.           // finalize transfer
  1813.           if (back[i].r.statuscode>0) {
  1814.             back_finalize(opt,cache,back,i);
  1815.           }
  1816.         }
  1817.       }
  1818. #endif
  1819.       else if (back[i].status==1001) {  // ftp ready
  1820.         back[i].status=0;
  1821.         // finalize transfer
  1822.         if (back[i].r.statuscode>0) {
  1823.           back_finalize(opt,cache,back,i);
  1824.         }
  1825.       }
  1826.       else if ((back[i].status>0) && (back[i].status<1000)) {  // en rΘception http
  1827.         int dispo=0;
  1828.         
  1829.         // vΘrifier l'existance de timeout-check
  1830.         if (!gestion_timeout)
  1831.           if (back[i].timeout>0)
  1832.             gestion_timeout=1;
  1833.           
  1834.           // donnΘes dispo?
  1835.           //## if (back[i].url_adr[0]!=lOCAL_CHAR)
  1836.           if (!back[i].r.is_file) {
  1837.             dispo=FD_ISSET(back[i].r.soc,&fds);
  1838.           }
  1839.           else
  1840.             dispo=1;
  1841.  
  1842.           // Check transfer rate!
  1843.           if (!max_read_bytes)
  1844.             dispo=0;                // limit transfer rate
  1845.           
  1846.           if (dispo) {    // donnΘes dispo
  1847.             LLint retour_fread;
  1848.             busy_recv=1;    // on rΘcupΦre encore
  1849. #if BDEBUG==1
  1850.             printf("..data available on socket %d\n",back[i].r.soc);
  1851. #endif
  1852.  
  1853.             
  1854.             // range size hack old location
  1855.  
  1856. #if HTS_DIRECTDISK
  1857.             // Court-circuit:
  1858.             // Peut-on stocker le fichier directement sur disque?
  1859.             // Ahh que ca serait vachement mieux et que ahh que la mΘmoire vous dit merci!
  1860.             if (back[i].status) {
  1861.               if (back[i].r.is_write==0) {  // mode mΘmoire
  1862.                 if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  1863.                   if (!back[i].testmode) {  // pas mode test
  1864.                     if (strnotempty(back[i].url_sav)) {
  1865.                       if (strcmp(back[i].url_fil,"/robots.txt")) {
  1866.                         if (back[i].r.statuscode==200) {  // 'OK'
  1867.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)
  1868.                             ) {    // pas HTML
  1869.                             if (opt->getmode&2) {    // on peut ecrire des non html
  1870.                               int fcheck=0;
  1871.                               back[i].r.is_write=1;    // Θcrire
  1872.                               if (back[i].r.compressed
  1873.                                 &&
  1874.                                 /* .gz are *NOT* depacked!! */
  1875.                                 (strfield(get_ext(back[i].url_sav),"gz") == 0)
  1876.                                 ) {
  1877.                                 back[i].tmpfile_buffer[0]='\0';
  1878.                                 back[i].tmpfile=tmpnam(back[i].tmpfile_buffer);
  1879.                                 if (back[i].tmpfile != NULL && back[i].tmpfile[0])
  1880.                                   back[i].r.out=fopen(back[i].tmpfile,"wb");
  1881.                               } else {
  1882.                                 back[i].r.compressed=0;
  1883.                                 back[i].r.out=filecreate(back[i].url_sav);
  1884.                               }
  1885.                               if (back[i].r.out==NULL) {
  1886.                                 if ((fcheck=check_fatal_io_errno())) {
  1887.                                   opt->state.exit_xh=-1;   /* fatal error */
  1888.                                 }
  1889.                               }
  1890. #if HDEBUG
  1891.                               printf("direct-disk: %s\n",back[i].url_sav);
  1892. #endif
  1893.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  1894.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File received from net to disk: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1895.                               }
  1896.                               
  1897.                               if (back[i].r.out==NULL) {
  1898.                                 if (opt->errlog) {
  1899.                                   fspc(opt->errlog,"error");
  1900.                                   fprintf(opt->errlog,"Unable to save file %s : %s"LF,back[i].url_sav, strerror(errno));
  1901.                                   if (fcheck) {
  1902.                                     fspc(opt->errlog,"error");
  1903.                                     fprintf(opt->errlog,"* * Fatal write error, giving up"LF);
  1904.                                   }
  1905.                                   test_flush;
  1906.                                 }
  1907.                                 back[i].r.is_write=0;    // erreur, abandonner
  1908. #if HDEBUG
  1909.                                 printf("..error!\n");
  1910. #endif
  1911.                               }
  1912. #if HTS_WIN==0
  1913.                               else chmod(back[i].url_sav,HTS_ACCESS_FILE);      
  1914. #endif          
  1915.                             } else {  // on coupe tout!
  1916.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  1917.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1918.                               }
  1919.                               back[i].status=0;  // terminΘ
  1920.                               if (!back[i].testmode)
  1921.                                 back[i].r.statuscode=-10;    // EUHH CANCEL
  1922.                               else
  1923.                                 back[i].r.statuscode=-10;    // "TEST OK"
  1924.                               if (back[i].r.soc!=INVALID_SOCKET) {
  1925. #if HTS_DEBUG_CLOSESOCK
  1926.                                 DEBUG_W("back_wait(3): deletehttp\n");
  1927. #endif
  1928.                                 deletehttp(&back[i].r);
  1929.                               }
  1930.                               back[i].r.soc=INVALID_SOCKET;
  1931.                             }
  1932.                           }
  1933.                         }
  1934.                       }
  1935.                     }
  1936.                   }
  1937.                 }
  1938.               }
  1939.             }
  1940. #endif              
  1941.  
  1942.             // rΘception de donnΘes depuis socket ou fichier
  1943.             if (back[i].status) {
  1944.               if (back[i].status==99)  // recevoir par bloc de lignes
  1945.                 retour_fread=http_xfread1(&(back[i].r),0);
  1946.               else if (back[i].status==98 || back[i].status==97) { // recevoir longueur chunk en hexa caractΦre par caractΦre
  1947.                 // backuper pour lire dans le buffer chunk
  1948.                 htsblk r;
  1949.                 memcpy(&r, &(back[i].r), sizeof(htsblk));
  1950.                 back[i].r.is_write=0;                   // mΘmoire
  1951.                 back[i].r.adr=back[i].chunk_adr;        // adresse
  1952.                 back[i].r.size=back[i].chunk_size;      // taille taille chunk
  1953.                 back[i].r.totalsize=-1;                 // total inconnu
  1954.                 back[i].r.out=NULL;
  1955.                 back[i].r.is_file=0;
  1956.                 //
  1957.                 // ligne par ligne
  1958.                 retour_fread=http_xfread1(&(back[i].r),-1);
  1959.                 // modifier et restaurer
  1960.                 back[i].chunk_adr=back[i].r.adr;        // adresse
  1961.                 back[i].chunk_size=back[i].r.size;      // taille taille chunk
  1962.                 memcpy(&(back[i].r), &r, sizeof(htsblk));    // restaurer vΘritable r
  1963.               }
  1964.               else if (back[i].is_chunk) {         // attention chunk, limiter taille α lire
  1965. #if CHUNKDEBUG==1
  1966.                 printf("[%d] read %d bytes\n",(int)back[i].r.soc,(int)min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  1967. #endif
  1968.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  1969.               } else              
  1970.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) max_read_bytes);
  1971.                 // retour_fread=http_fread1(&(back[i].r));
  1972.             } else
  1973.               retour_fread=-1;                    // interruption ou annulation interne (peut ne pas Ωtre une erreur)
  1974.             
  1975.             // Si rΘception chunk, tester si on est pas α la fin!
  1976.             if (back[i].status==1) {
  1977.               if (back[i].is_chunk) {     // attendre prochain chunk
  1978.                 if (back[i].r.size==back[i].r.totalsize) {      // fin chunk!
  1979.                   //printf("chunk end at %d\n",back[i].r.size);
  1980.                   back[i].status=97;  /* fetch ending CRLF */
  1981.                   if (back[i].chunk_adr!=NULL) { 
  1982.                     freet(back[i].chunk_adr); 
  1983.                     back[i].chunk_adr=NULL; 
  1984.                   } 
  1985.                   back[i].chunk_size=0;
  1986.                   retour_fread=0;       // pas d'erreur
  1987. #if CHUNKDEBUG==1
  1988.                   printf("[%d] waiting for current chunk CRLF..\n",(int)back[i].r.soc);
  1989. #endif
  1990.                 }
  1991.               } else if (back[i].r.keep_alive) {
  1992.                 if (back[i].r.size==back[i].r.totalsize) {      // fin!
  1993.                   retour_fread=-1;       // end
  1994.                 }
  1995.               }
  1996.             }
  1997.             
  1998.             if (retour_fread < 0) {    // fin rΘception
  1999.               back[i].status=0;    // terminΘ
  2000.              /*KA back[i].r.soc=INVALID_SOCKET; */
  2001. #if CHUNKDEBUG==1
  2002.               if (back[i].is_chunk)
  2003.                 printf("[%d] must be the last chunk for %s (connection closed) - %d/%d\n",(int)back[i].r.soc,back[i].url_fil,back[i].r.size,back[i].r.totalsize);
  2004. #endif
  2005.               //if ((back[i].r.statuscode==-1) && (strnotempty(back[i].r.msg)==0)) {
  2006.               if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) {
  2007. #if HDEBUG
  2008.                 printf("error interruped: %s\n",back[i].r.adr);
  2009. #endif        
  2010.                 if (back[i].r.size>0)
  2011.                   strcatbuff(back[i].r.msg,"Interrupted transfer");
  2012.                 else
  2013.                   strcatbuff(back[i].r.msg,"No data (connection closed)");
  2014.                 back[i].r.statuscode=-4;
  2015.               }
  2016.  
  2017.               // Close socket
  2018.               if (back[i].r.soc!=INVALID_SOCKET) {
  2019. #if HTS_DEBUG_CLOSESOCK
  2020.                 DEBUG_W("back_wait(4): deletehttp\n");
  2021. #endif
  2022.                 /*KA deletehttp(&back[i].r);*/
  2023.                 back_maydeletehttp(opt, cache, back, back_max, i);
  2024.               }
  2025.  
  2026.               // finalize transfer
  2027.               if (back[i].r.statuscode>0) {
  2028.                 back_finalize(opt,cache,back,i);
  2029.               }
  2030.  
  2031.               if (back[i].r.totalsize>0) {    // tester totalsize
  2032.               //if ((back[i].r.totalsize>0) && (back[i].status==99)) {    // tester totalsize
  2033.                 if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2034.                   if (!opt->tolerant) {
  2035.                     //#if HTS_CL_IS_FATAL
  2036.                     deleteaddr(&back[i].r);
  2037.                     if (back[i].r.size<back[i].r.totalsize)
  2038.                       back[i].r.statuscode=-4;        // recatch
  2039.                     sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize);
  2040.                   } else {
  2041.                     //#else
  2042.                     // Un warning suffira..
  2043.                     if (cache->errlog!=NULL) {
  2044.                       fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2045.                     }
  2046.                     //#endif
  2047.                   }
  2048.                 }
  2049.               }
  2050. #if BDEBUG==1
  2051.               printf("transfer ok\n");
  2052. #endif
  2053.             } else if (retour_fread > 0) {    // pas d'erreur de rΘception et data
  2054.               if (back[i].timeout>0) {    // refresh timeout si besoin est
  2055.                 back[i].timeout_refresh=time_local();
  2056.               }
  2057.  
  2058.               // Traitement des en tΩtes chunks ou en tΩtes
  2059.               if (back[i].status==98 || back[i].status==97) {        // rΘception taille chunk en hexa (  aprΦs les en tΩtes, peut ne pas
  2060.                 if (back[i].chunk_size > 0 && back[i].chunk_adr[back[i].chunk_size-1]==10) {
  2061.                   int chunk_size=-1;
  2062.                   char chunk_data[64];
  2063.                   if (back[i].chunk_size<32) {      // pas trop gros
  2064.                     char* chstrip=back[i].chunk_adr;
  2065.                     back[i].chunk_adr[ back[i].chunk_size-1]='\0';    // octet nul 
  2066.                     // skip leading spaces or cr
  2067.                     while(isspace(*chstrip)) chstrip++;
  2068.                     chunk_data[0] = '\0';
  2069.                     strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2);
  2070.                     // strip chunk-extension
  2071.                     while( (chstrip = strchr(chunk_data, ';'))) *chstrip='\0';
  2072.                     while( (chstrip = strchr(chunk_data, ' '))) *chstrip='\0';
  2073.                     while( (chstrip = strchr(chunk_data, '\r'))) *chstrip='\0';
  2074. #if CHUNKDEBUG==1
  2075.                     printf("[%d] chunk received and read: %s\n",(int)back[i].r.soc,chunk_data);
  2076. #endif
  2077.                     if (back[i].r.totalsize<0)
  2078.                       back[i].r.totalsize=0;        // initialiser α 0
  2079.                     if (back[i].status==98) {   // "real" chunk
  2080.                       if (sscanf(chunk_data,"%x",&chunk_size) == 1) {
  2081.                         if (chunk_size > 0)
  2082.                           back[i].chunk_blocksize = chunk_size;  /* the data block chunk size */
  2083.                         else
  2084.                           back[i].chunk_blocksize = -1;  /* ending */
  2085.                         back[i].r.totalsize+=chunk_size;    // noter taille
  2086.                         back[i].r.adr=(char*) realloct(back[i].r.adr,(INTsys) back[i].r.totalsize + 1);
  2087.                         if (!back[i].r.adr) {
  2088.                           if (cache->errlog!=NULL) {
  2089.                             fprintf(cache->errlog,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2090.                           }
  2091.                         }
  2092. #if CHUNKDEBUG==1
  2093.                         printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize);
  2094. #endif
  2095.                       } else {
  2096.                         if (cache->errlog!=NULL) {
  2097.                           fprintf(cache->errlog,"Warning: Illegal chunk (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2098.                         }
  2099.                       }
  2100.                     } else {   /* back[i].status==97 : just receiving ending CRLF after data */
  2101.                       if (chunk_data[0] == '\0') {
  2102.                         if (back[i].chunk_blocksize > 0)
  2103.                           chunk_size=(int)back[i].chunk_blocksize;  /* recent data chunk size */
  2104.                         else if (back[i].chunk_blocksize == -1)
  2105.                           chunk_size=0;                        /* ending chunk */
  2106.                         else
  2107.                           chunk_size=1;                        /* fake positive size for 1st chunk history */
  2108. #if CHUNKDEBUG==1
  2109.                         printf("[%d] chunk CRLF seen\n", (int)back[i].r.soc);
  2110. #endif
  2111.                       } else {
  2112.                         if (cache->errlog!=NULL) {
  2113.                           fprintf(cache->errlog,"Warning: Illegal chunk CRLF (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2114.                         }
  2115. #if CHUNKDEBUG==1
  2116.                         printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int)back[i].r.soc, chunk_data);
  2117. #endif
  2118.                       }
  2119.                     }
  2120.                   } else {                                  
  2121.                     if (cache->errlog!=NULL) {
  2122.                       fprintf(cache->errlog,"Warning: Chunk too big ("LLintP") for %s%s"LF,(LLint)back[i].chunk_size,back[i].url_adr,back[i].url_fil);
  2123.                     }
  2124.                   }
  2125.                     
  2126.                   // ok, continuer sur le body
  2127.                     
  2128.                   // si chunk non nul continuer (ou commencer)
  2129.                   if (back[i].status==97 && chunk_size > 0) {
  2130.                     back[i].status = 98;  /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */
  2131. #if CHUNKDEBUG==1
  2132.                     printf("[%d] waiting for next chunk\n", (int)back[i].r.soc);
  2133. #endif
  2134.                   } else if (back[i].status==98 && chunk_size == 0) {  /* final chunk */
  2135.                     back[i].status=97;  /* final CRLF */
  2136. #if CHUNKDEBUG==1
  2137.                     printf("[%d] waiting for final CRLF (chunk)\n", (int)back[i].r.soc);
  2138. #endif
  2139.                   } else if (back[i].status==98 && chunk_size >= 0) {  /* will fetch data now */
  2140.                     back[i].status=1;     // continuer body    
  2141. #if CHUNKDEBUG==1
  2142.                     printf("[%d] waiting for body (chunk)\n", (int)back[i].r.soc);
  2143. #endif
  2144.                   } else {                /* zero-size-chunk-CRLF (end) or error */
  2145. #if CHUNKDEBUG==1
  2146.                     printf("[%d] chunk end, total: %d\n",(int)back[i].r.soc,back[i].r.size);
  2147. #endif
  2148.                     /* End */
  2149.                     //if (back[i].status==97) {
  2150.                     back[i].status=0;     // fin  
  2151.                     //}
  2152.  
  2153.                     // finalize transfer
  2154.                     back_finalize(opt,cache,back,i);
  2155.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2156. #if HTS_DEBUG_CLOSESOCK
  2157.                       DEBUG_W("back_wait(5): deletehttp\n");
  2158. #endif
  2159.                       /* Error */
  2160.                       if (chunk_size < 0) {
  2161.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2162.                         deleteaddr(&back[i].r);
  2163.                         back[i].r.statuscode=-1;
  2164.                         strcpybuff(back[i].r.msg,"Invalid chunk");
  2165. #if CHUNKDEBUG==1
  2166.                         printf("[%d] chunk error\n", (int)back[i].r.soc);
  2167. #endif
  2168.                       } else /* if chunk_size == 0 */ {
  2169. #if CHUNKDEBUG==1
  2170.                         printf("[%d] all chunks now received\n", (int)back[i].r.soc);
  2171. #endif
  2172.                           
  2173.                         /* Tester totalsize en fin de chunk */
  2174.                         if ((back[i].r.totalsize>0)) {    // tester totalsize
  2175.                           if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2176. #if HTS_CL_IS_FATAL
  2177.                             deleteaddr(&back[i].r);
  2178.                             back[i].r.statuscode=-1;
  2179.                             strcpybuff(back[i].r.msg,"Incorrect length");
  2180. #else
  2181.                             // Un warning suffira..
  2182.                             if (cache->errlog!=NULL) {
  2183.                               fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2184.                             }
  2185. #endif
  2186.                           }
  2187.                         }
  2188.                           
  2189.                         /* Oops, trailers! */
  2190.                         if (back[i].r.keep_alive_trailers) {
  2191.                           /* fixme (not yet supported) */
  2192.                         }
  2193.                           
  2194.                       }
  2195.                         
  2196.                         
  2197.                     }
  2198.                   }
  2199.                     
  2200.                   // effacer buffer (chunk en tete)
  2201.                   if (back[i].chunk_adr!=NULL) {
  2202.                     freet(back[i].chunk_adr);
  2203.                     back[i].chunk_adr=NULL;
  2204.                     back[i].chunk_size=0;
  2205.                     // NO! xxback[i].chunk_blocksize = 0;
  2206.                   }
  2207.                   
  2208.                 } // taille buffer chunk > 1 && LF
  2209.                 //
  2210.               } else if (back[i].status==99) {        // en tΩtes (avant le chunk si il est prΘsent)
  2211.                 //
  2212.                 if (back[i].r.size>=2) {
  2213.                   // double LF
  2214.                   if (
  2215.                     ((back[i].r.adr[back[i].r.size-1]==10) && (back[i].r.adr[back[i].r.size-2]==10)) 
  2216.                     ||
  2217.                     (back[i].r.adr[0] == '<')    /* bogus server */
  2218.                     ) {
  2219.                     char rcvd[2048];
  2220.                     int ptr=0;
  2221.                     int noFreebuff=0;
  2222.                     
  2223. #if BDEBUG==1
  2224.                     printf("..ok, header received\n");
  2225. #endif
  2226.                     
  2227.                     
  2228.                     /* Hack for zero-length headers */
  2229.                     if (back[i].status != 0 && back[i].r.adr[0] != '<') {
  2230.                       
  2231.                       // ----------------------------------------
  2232.                       // traiter en-tΩte!
  2233.                       // status-line α rΘcupΘrer
  2234.                       ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2235.                       if (strnotempty(rcvd)==0) {
  2236.                         /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */
  2237.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2238.                       }
  2239.                       
  2240.                       // traiter status-line
  2241.                       treatfirstline(&back[i].r,rcvd);
  2242.                       
  2243. #if HDEBUG
  2244.                       printf("(Buffer) Status-Code=%d\n",back[i].r.statuscode);
  2245. #endif
  2246.                       if (_DEBUG_HEAD) {
  2247.                         if (ioinfo) {
  2248.                           fprintf(ioinfo,"[%d] response for %s%s:\r\ncode=%d\r\n",
  2249.                             back[i].r.debugid, jump_identification(back[i].url_adr),back[i].url_fil,back[i].r.statuscode);
  2250.                           fprintfio(ioinfo,back[i].r.adr,">>> ");
  2251.                           fprintf(ioinfo,"\r\n");
  2252.                           fflush(ioinfo);
  2253.                         }                    // en-tΩte
  2254.                       }
  2255.                       
  2256.                       // header // ** !attention! HTTP/0.9 non supportΘ
  2257.                       do {
  2258.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);          
  2259. #if HDEBUG
  2260.                         printf("(buffer)>%s\n",rcvd);      
  2261. #endif
  2262.                         /*
  2263.                         if (_DEBUG_HEAD) {
  2264.                         if (ioinfo) {
  2265.                         fprintf(ioinfo,"(buffer)>%s\r\n",rcvd);      
  2266.                         fflush(ioinfo);
  2267.                         }
  2268.                         }
  2269.                         */
  2270.                         
  2271.                         if (strnotempty(rcvd))
  2272.                           treathead(opt->cookie,back[i].url_adr,back[i].url_fil,&back[i].r,rcvd);  // traiter
  2273.                         
  2274.                         // parfois les serveurs buggΘs renvoient un content-range avec un 200
  2275.                         if (back[i].r.statuscode==200)  // 'OK'
  2276.                           if (strfield(rcvd,"content-range:"))  // Avec un content-range: relisez les RFC..
  2277.                             back[i].r.statuscode=206;    // FORCER A 206 !!!!!
  2278.                           
  2279.                       } while(strnotempty(rcvd));
  2280.                       // ----------------------------------------                    
  2281.  
  2282.                     } else {
  2283.                       // assume text/html, OK
  2284.                       treatfirstline(&back[i].r, back[i].r.adr);
  2285.                       noFreebuff=1;
  2286.                     }
  2287.                       
  2288.                     // Callback
  2289. #if HTS_ANALYSTE
  2290.                     if (hts_htmlcheck_receivehead != NULL) {
  2291.                       int test_head=hts_htmlcheck_receivehead(back[i].r.adr, back[i].url_adr, back[i].url_fil, back[i].referer_adr, back[i].referer_fil, &back[i].r);
  2292.                       if (test_head!=1) {
  2293.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  2294.                           fspc(opt->log,"warning"); fprintf(opt->log,"External wrapper aborted transfer, breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2295.                         }
  2296.                         back[i].status=0;  // FINI
  2297.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2298.                         strcpybuff(back[i].r.msg,"External wrapper aborted transfer");
  2299.                         back[i].r.statuscode = -1;
  2300.                       }
  2301.                     }
  2302. #endif
  2303.  
  2304.                     // Free headers memory now
  2305.                     // Actually, save them for informational purpose
  2306.                     if (!noFreebuff) {
  2307.                       char* block = back[i].r.adr;
  2308.                       back[i].r.adr = NULL;
  2309.                       deleteaddr(&back[i].r);
  2310.                       back[i].r.headers = block;
  2311.                     }                  
  2312.                     
  2313.                     /* 
  2314.                     Status code and header-response hacks
  2315.                     */
  2316.  
  2317.                     
  2318.                     // Check response : 203 == 200
  2319.                     if (back[i].r.statuscode==203) {  // 'Non-Authoritative Information'
  2320.                       back[i].r.statuscode=200;       // forcer "OK"
  2321.                     } else if (back[i].r.statuscode == 100) {
  2322.                       back[i].status=99;
  2323.                       back[i].r.size=0;
  2324.                       back[i].r.totalsize=0;
  2325.                       back[i].chunk_size=0;
  2326.                       back[i].r.statuscode=-1;
  2327.                       back[i].r.msg[0]='\0';
  2328.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2329.                         fspc(opt->log,"debug"); fprintf(opt->log,"Status 100 detected for %s%s, continuing headers"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2330.                       }
  2331.                       continue;
  2332.                     }
  2333.                     
  2334.                     /*
  2335.                     Solve "false" 416 problems
  2336.                     */
  2337.                     if (back[i].r.statuscode==416) {  // 'Requested Range Not Satisfiable'
  2338.                       // Example:
  2339.                       // Range: bytes=2830-
  2340.                       // ->
  2341.                       // Content-Range: bytes */2830
  2342.                       if (back[i].range_req_size == back[i].r.crange) {
  2343.                         filenote(back[i].url_sav,NULL);
  2344.                         //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2345.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2346.                         back[i].status=0;    // READY
  2347.                         back[i].r.size=back[i].r.totalsize=back[i].range_req_size;
  2348.                         back[i].r.statuscode=304;     // NOT MODIFIED
  2349.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2350.                           fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (good 416 message), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2351.                         }
  2352.                       }
  2353.                     }
  2354.                     
  2355.                     // transform 406 into 200 ; we'll catch embedded links inside the choice page
  2356.                     if (back[i].r.statuscode==406) {  // 'Not Acceptable'
  2357.                       back[i].r.statuscode=200;
  2358.                     }
  2359.  
  2360.                     // Various hacks to limit re-transfers when updating a mirror
  2361.                     // Force update if same size detected
  2362.                     if (opt->sizehack) {
  2363.                       // We already have the file
  2364.                       // and ask the remote server for an update
  2365.                       // Some servers, especially dynamic pages severs, always
  2366.                       // answer that the page has been modified since last visit
  2367.                       // And answer with a 200 (OK) response, and the same page
  2368.                       // If the size is the same, and the option has been set, we assume
  2369.                       // that the file is identical - and therefore let's break the connection
  2370.                       if (back[i].is_update) {          // mise α jour
  2371.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2372.                           htsblk r = cache_read(opt,cache,back[i].url_adr,back[i].url_fil,NULL,NULL);    // lire entrΘe cache
  2373.                           if (r.statuscode == 200) {  // OK pas d'erreur cache
  2374.                             LLint len1,len2;
  2375.                             len1=r.totalsize;
  2376.                             len2=back[i].r.totalsize;
  2377.                             if (r.size>0)
  2378.                               len1=r.size;
  2379.                             if (len1>0) {
  2380.                               if (len1 == len2) {             // tailles identiques
  2381.                                 back[i].r.statuscode=304;     // forcer NOT MODIFIED
  2382.                                 deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2383.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2384.                                   fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2385.                                 }
  2386.                               }
  2387.                             }
  2388.                           } else {
  2389.                             if (opt->errlog!=NULL) {
  2390.                               fspc(opt->errlog,"warning"); fprintf(opt->errlog,"File seems complete (same size), but there was a cache read error: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2391.                             }
  2392.                           }
  2393.                           if (r.adr) {
  2394.                             freet(r.adr);
  2395.                           }
  2396.                         }
  2397.                       }
  2398.                     }
  2399.                     
  2400.                     // Various hacks to limit re-transfers when updating a mirror
  2401.                     // Detect already downloaded file (with another browser, for example)
  2402.                     if (opt->sizehack) {
  2403.                       if (!back[i].is_update) {          // mise α jour
  2404.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2405.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) {    // not HTML
  2406.                             if (strnotempty(back[i].url_sav)) {  // target found
  2407.                               int size = fsize(back[i].url_sav);  // target size
  2408.                               if (size >= 0) {
  2409.                                 if (back[i].r.totalsize == size) {  // same size!
  2410.                                   deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2411.                                   back[i].status=0;    // READY
  2412.                                   back[i].r.size=back[i].r.totalsize;
  2413.                                   filenote(back[i].url_sav,NULL);
  2414.                                   //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2415.                                   back[i].r.statuscode=304;     // NOT MODIFIED
  2416.                                   if ((opt->debug>1) && (opt->log!=NULL)) {
  2417.                                     fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size file discovered), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2418.                                   }
  2419.                                 }
  2420.                               }
  2421.                             }
  2422.                           }
  2423.                         }
  2424.                       }
  2425.                     }
  2426.                     
  2427.                     // Various hacks to limit re-transfers when updating a mirror
  2428.                     // Detect bad range: header
  2429.                     if (opt->sizehack) {
  2430.                       // We have request for a partial file (with a 'Range: NNN-' header)
  2431.                       // and received a complete file notification (200), with 'Content-length: NNN'
  2432.                       // it might be possible that we had the complete file
  2433.                       // this is the case in *most* cases, so break the connection
  2434.                       if (back[i].r.is_write==0) {  // mode mΘmoire
  2435.                         if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  2436.                           if (!back[i].testmode) {  // pas mode test
  2437.                             if (strnotempty(back[i].url_sav)) {
  2438.                               if (strcmp(back[i].url_fil,"/robots.txt")) {
  2439.                                 if (back[i].r.statuscode==200) {  // 'OK'
  2440.                                   if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) {    // pas HTML
  2441.                                     if (back[i].r.statuscode==200) {      // "OK"
  2442.                                       if (back[i].range_req_size>0) {     // but Range: requested
  2443.                                         if (back[i].range_req_size == back[i].r.totalsize) {    // And same size
  2444. #if HTS_DEBUG_CLOSESOCK
  2445.                                           DEBUG_W("back_wait(skip_range): deletehttp\n");
  2446. #endif
  2447.                                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2448.                                           back[i].status=0;    // READY
  2449.                                           back[i].r.size=back[i].r.totalsize;
  2450.                                           filenote(back[i].url_sav,NULL);
  2451.                                           //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2452.                                           back[i].r.statuscode=304;     // NOT MODIFIED
  2453.                                           if ((opt->debug>1) && (opt->log!=NULL)) {
  2454.                                             fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (reget failed), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2455.                                           }
  2456.                                         }
  2457.                                       }
  2458.                                     }
  2459.                                     
  2460.                                   }
  2461.                                 }
  2462.                               }
  2463.                             }
  2464.                           }
  2465.                         }
  2466.                       }
  2467.                     }
  2468.                     // END - Various hacks to limit re-transfers when updating a mirror
  2469.  
  2470.                     /* 
  2471.                     End of status code and header-response hacks
  2472.                     */
  2473.  
  2474.                     
  2475.                     
  2476.                     /* Interdiction taille par le wizard? */
  2477.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2478.                       if (!back_checksize(opt,&back[i],1)) {
  2479.                         back[i].status=0;  // FINI
  2480.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2481.                         if (!back[i].testmode)
  2482.                           strcpybuff(back[i].r.msg,"File too big");
  2483.                         else
  2484.                           strcpybuff(back[i].r.msg,"Test: File too big");
  2485.                       }
  2486.                     }
  2487.                     
  2488.                     /* sinon, continuer */
  2489.                     /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok rΘcupΘrer body? */
  2490.                     // head: terminΘ
  2491.                     if (back[i].head_request) {
  2492.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2493.                         fspc(opt->log,"debug"); fprintf(opt->log,"Tested file: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2494.                       }
  2495. #if HTS_DEBUG_CLOSESOCK
  2496.                       DEBUG_W("back_wait(head request): deletehttp\n");
  2497. #endif
  2498.                       // Couper connexion
  2499.                       if (!back[i].http11) {    /* NO KA */
  2500.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2501.                       }
  2502.                       back[i].status=0;  // terminΘ
  2503.                     }
  2504.                     // traiter une Θventuelle erreur 304 (cache α jour utilisable)
  2505.                     else if (back[i].r.statuscode==304) {  // document α jour dans le cache
  2506.                       // lire dans le cache
  2507.                       // ** NOTE: pas de vΘrif de la taille ici!!
  2508. #if HTS_DEBUG_CLOSESOCK
  2509.                       DEBUG_W("back_wait(file is not modified): deletehttp\n");
  2510. #endif
  2511.                       /* clear everything but connection: switch, close, and reswitch */
  2512.                       {
  2513.                         htsblk tmp;
  2514.                         memset(&tmp, 0, sizeof(tmp));
  2515.                         back_connxfr(&back[i].r, &tmp);
  2516.                         back[i].r=cache_read(opt,cache,back[i].url_adr,back[i].url_fil,back[i].url_sav,back[i].location_buffer);
  2517.                         back[i].r.location=back[i].location_buffer;
  2518.                         back_connxfr(&tmp,&back[i].r);
  2519.                       }
  2520.  
  2521.                       // hack:
  2522.                       // In case of 'if-unmodified-since' hack, a 304 status can be sent
  2523.                       // then, force 'ok' status
  2524.                       if (back[i].r.statuscode == -1) {
  2525.                         if (fexist(back[i].url_sav)) {
  2526.                           back[i].r.statuscode=200;     // OK
  2527.                           strcpybuff(back[i].r.msg, "OK (cached)");
  2528.                           back[i].r.is_file=1;
  2529.                           back[i].r.totalsize = back[i].r.size = fsize(back[i].url_sav);
  2530.                           get_httptype(back[i].r.contenttype, back[i].url_sav, 1);
  2531.                           if ((opt->debug>0) && (opt->log!=NULL)) {
  2532.                             fspc(opt->log,"debug"); fprintf(opt->log,"Not-modified status without cache guessed: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2533.                           }
  2534.                         }
  2535.                       }
  2536.  
  2537.                       // Status is okay?
  2538.                       if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture
  2539.                         back[i].status=0;         // OK prΩt
  2540.                         back[i].r.notmodified=1;  // NON modifiΘ!
  2541.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  2542.                           fspc(opt->log,"debug"); fprintf(opt->log,"File loaded after test from cache: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2543.                         }
  2544.  
  2545.                         // finalize
  2546.                         if (back[i].r.statuscode>0) {
  2547.                           back_finalize(opt,cache,back,i);
  2548.                         }
  2549.                         
  2550. #if DEBUGCA
  2551.                         printf("..document α jour aprΦs requΦte: %s%s\n",back[i].url_adr,back[i].url_fil);
  2552. #endif
  2553.                         
  2554.                         //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
  2555.                       } else {  // erreur
  2556.                         back[i].status=0;  // terminΘ
  2557.                         //printf("erreur cache\n");
  2558.                         
  2559.                       } 
  2560.                       
  2561. /********** NO - must complete the body! ********** */
  2562. #if 0
  2563.                     } else if ((back[i].r.statuscode==301)
  2564.                       || (back[i].r.statuscode==302)
  2565.                       || (back[i].r.statuscode==303)
  2566.                       || (back[i].r.statuscode==307)
  2567.                       || (back[i].r.statuscode==412)
  2568.                       || (back[i].r.statuscode==416)
  2569.                       ) {   // Ne pas prendre le html, erreurs connues et gΘrΘes
  2570. #if HTS_DEBUG_CLOSESOCK
  2571.                       DEBUG_W("back_wait(301,302,303,307,412,416..): deletehttp\n");
  2572. #endif
  2573.                       // Couper connexion
  2574.                       /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/
  2575.                       back_maydeletehttp(opt, cache, back, back_max, i);
  2576.  
  2577.                       back[i].status=0;  // terminΘ
  2578.                       // finalize
  2579.                       if (back[i].r.statuscode>0) {
  2580.                         back_finalize(opt,cache,back,i);
  2581.                       }
  2582. #endif
  2583. /********** **************************** ********** */
  2584.                     } else {    // il faut aller le chercher
  2585.                       
  2586.                       // effacer buffer (requΦte)
  2587.                       if (!noFreebuff) {
  2588.                         deleteaddr(&back[i].r);
  2589.                         back[i].r.size=0;
  2590.                       }
  2591.                       
  2592.                       // traiter 206 (partial content)
  2593.                       // xxc SI CHUNK VERIFIER QUE CA MARCHE??
  2594.                       if (back[i].r.statuscode==206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
  2595.                         LLint sz=fsize(back[i].url_sav);
  2596. #if HDEBUG
  2597.                         printf("partial content: "LLintP" on disk..\n",(LLint)sz);
  2598. #endif
  2599.                         if (sz>=0) {
  2600.                           if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_sav)) {    // pas HTML
  2601.                             if (opt->getmode&2) {    // on peut ecrire des non html  **sinon ben euhh sera interceptΘ plus loin, donc rap sur ce qui va sortir**
  2602.                               filenote(back[i].url_sav,NULL);    // noter fichier comme connu
  2603.                               back[i].r.out=fopen(fconv(back[i].url_sav),"ab");  // append
  2604.                               if (back[i].r.out) {
  2605.                                 back[i].r.is_write=1;    // Θcrire
  2606.                                 back[i].r.size=sz;    // dΘja Θcrit
  2607.                                 back[i].r.statuscode=200;  // Forcer 'OK'
  2608.                                 if (back[i].r.totalsize>0)
  2609.                                   back[i].r.totalsize+=sz;    // plus en fait
  2610.                                 fseek(back[i].r.out,0,SEEK_END);  // α la fin
  2611. #if HDEBUG
  2612.                                 printf("continue interrupted file\n");
  2613. #endif
  2614.                               } else {    // On est dans la m**
  2615.                                 back[i].status=0;  // terminΘ (voir plus loin)
  2616.                                 strcpybuff(back[i].r.msg,"Can not open partial file");
  2617.                               }
  2618.                             }
  2619.                           } else {    // mΘmoire
  2620.                             FILE* fp=fopen(fconv(back[i].url_sav),"rb");
  2621.                             if (fp) {
  2622.                               LLint alloc_mem=sz + 1;
  2623.                               if (back[i].r.totalsize>0)
  2624.                                 alloc_mem+=back[i].r.totalsize;            // AJOUTER RESTANT!
  2625.                               if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) alloc_mem)) ) {
  2626.                                 back[i].r.size=sz;
  2627.                                 if (back[i].r.totalsize>0)
  2628.                                   back[i].r.totalsize+=sz;    // plus en fait
  2629.                                 if (( fread(back[i].r.adr,1,(INTsys)sz,fp)) != sz) {
  2630.                                   back[i].status=0;  // terminΘ (voir plus loin)
  2631.                                   strcpybuff(back[i].r.msg,"Can not read partial file");
  2632.                                 } else {
  2633.                                   back[i].r.statuscode=200;  // Forcer 'OK'
  2634. #if HDEBUG
  2635.                                   printf("continue in mem interrupted file\n");
  2636. #endif
  2637.                                 }
  2638.                               } else {
  2639.                                 back[i].status=0;  // terminΘ (voir plus loin)
  2640.                                 strcpybuff(back[i].r.msg,"No memory for partial file");
  2641.                               }
  2642.                               fclose(fp);
  2643.                             } else {  // Argh.. 
  2644.                               back[i].status=0;  // terminΘ (voir plus loin)
  2645.                               strcpybuff(back[i].r.msg,"Can not open partial file");
  2646.                             }
  2647.                           }
  2648.                         } else {    // Non trouvΘ??
  2649.                           back[i].status=0;  // terminΘ (voir plus loin)
  2650.                           strcpybuff(back[i].r.msg,"Can not find partial file");
  2651.                         }
  2652.                         // Erreur?
  2653.                         if (back[i].status==0) {
  2654.                           if (back[i].r.soc!=INVALID_SOCKET) {
  2655. #if HTS_DEBUG_CLOSESOCK
  2656.                             DEBUG_W("back_wait(206 solve problems): deletehttp\n");
  2657. #endif
  2658.                             deletehttp(&back[i].r);
  2659.                           }
  2660.                           back[i].r.soc=INVALID_SOCKET;
  2661.                           //back[i].r.statuscode=206;  ????????
  2662.                           back[i].r.statuscode=-5;
  2663.                           if (strnotempty(back[i].r.msg))
  2664.                             strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)");
  2665.                         }
  2666.                       }
  2667.                       
  2668.                       if (back[i].status!=0) {  // non terminΘ (erreur)
  2669.                         if (!back[i].testmode) {    // fichier normal
  2670.                           
  2671.                           if (back[i].r.empty && back[i].r.statuscode==200) {  // empty response
  2672.                             // Couper connexion
  2673.                             deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2674.                             back[i].status=0;  // terminΘ
  2675.                             if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) 2)) ) {
  2676.                               back[i].r.adr[0] = 0;
  2677.                             }
  2678.                             back_finalize(opt,cache,back,i);
  2679.                           }
  2680.                           else if (!back[i].r.is_chunk) {    // pas de chunk
  2681.                             //if (back[i].r.http11!=2) {    // pas de chunk
  2682.                             back[i].is_chunk=0;
  2683.                             back[i].status=1;     // start body
  2684.                           } else {
  2685. #if CHUNKDEBUG==1
  2686.                             printf("[%d] chunk encoding detected %s..\n",(int)back[i].r.soc, back[i].url_fil);
  2687. #endif
  2688.                             back[i].is_chunk=1;
  2689.                             back[i].chunk_adr=NULL;
  2690.                             back[i].chunk_size=0;
  2691.                             back[i].chunk_blocksize=0;
  2692.                             back[i].status=98;    // start body wait chunk
  2693.                             back[i].r.totalsize=0;   /* devalidate size! (rfc) */
  2694.                           }
  2695.                           if (back[i].rateout>0) {
  2696.                             back[i].rateout_time=time_local();  // refresh pour transfer rate
  2697.                           }
  2698. #if HDEBUG
  2699.                           printf("(buffer) start body!\n");
  2700. #endif
  2701.                         } else {     // mode test, ne pas passer en 1!!
  2702.                           back[i].status=0;    // READY
  2703. #if HTS_DEBUG_CLOSESOCK
  2704.                           DEBUG_W("back_wait(test ok): deletehttp\n");
  2705. #endif
  2706.                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2707.                           if (back[i].r.statuscode==200) {
  2708.                             strcpybuff(back[i].r.msg,"Test: OK");
  2709.                             back[i].r.statuscode=-10;    // test rΘussi
  2710.                           }
  2711.                           else {    // test a ΘchouΘ, on ne change rien sauf que l'erreur est α titre indicatif
  2712.                             char tempo[1000];
  2713.                             strcpybuff(tempo,back[i].r.msg);
  2714.                             strcpybuff(back[i].r.msg,"Test: ");
  2715.                             strcatbuff(back[i].r.msg,tempo);
  2716.                           }
  2717.                           
  2718.                         }
  2719.                       }
  2720.                       
  2721.                       } 
  2722.                       
  2723.                       /*}*/
  2724.                       
  2725.                   }  // si LF
  2726.                 }  // r.size>2
  2727.               }  // si == 99
  2728.               
  2729.             } // si pas d'erreurs
  2730. #if BDEBUG==1
  2731.             printf("bytes overall: %d\n",back[i].r.size);
  2732. #endif
  2733.           }  // donnΘes dispo
  2734.           
  2735.           // en cas d'erreur cl, supprimer Θventuel fichier sur disque
  2736. #if HTS_REMOVE_BAD_FILES
  2737.           if (back[i].status<0) {
  2738.             if (!back[i].testmode) {    // pas en test
  2739.               remove(back[i].url_sav);    // Θliminer fichier (endommagΘ)
  2740.               //printf("&& %s\n",back[i].url_sav);
  2741.             }
  2742.           }
  2743. #endif
  2744.  
  2745.           /* funny log for commandline users */
  2746.           //if (!opt->quiet) {  
  2747.           // petite animation
  2748.           if (opt->verbosedisplay==1) {
  2749.             if (back[i].status==0) {
  2750.               if (back[i].r.statuscode==200)
  2751.                 printf("* %s%s ("LLintP" bytes) - OK"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size);
  2752.               else
  2753.                 printf("* %s%s ("LLintP" bytes) - %d"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size,back[i].r.statuscode);
  2754.               fflush(stdout);
  2755.             }
  2756.           }
  2757.           //}
  2758.           
  2759.  
  2760.       } // status>0
  2761.     }  // for
  2762.     
  2763.     // vΘrifier timeouts
  2764.     if (gestion_timeout) {
  2765.       TStamp act;
  2766.       act=time_local();    // temps en secondes
  2767.       for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  2768.       // for(i=0;i<back_max;i++) {
  2769.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  2770.         if (back[i].status>0) {  // rΘception/connexion/..
  2771.           if (back[i].timeout>0) {
  2772.             //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
  2773.             if (((int) (act-back[i].timeout_refresh))>=back[i].timeout) {
  2774.               if (back[i].r.soc!=INVALID_SOCKET) {
  2775. #if HTS_DEBUG_CLOSESOCK
  2776.                 DEBUG_W("back_wait(timeout): deletehttp\n");
  2777. #endif
  2778.                 deletehttp(&back[i].r);
  2779.               }
  2780.               back[i].r.soc=INVALID_SOCKET;
  2781.               back[i].r.statuscode=-2;
  2782.               if (back[i].status==100)
  2783.                 strcpybuff(back[i].r.msg,"Connect Time Out");
  2784.               else if (back[i].status==101)
  2785.                 strcpybuff(back[i].r.msg,"DNS Time Out");
  2786.               else
  2787.                 strcpybuff(back[i].r.msg,"Receive Time Out");
  2788.               back[i].status=0;  // terminΘ
  2789.             } else if ((back[i].rateout>0) && (back[i].status<99)) {
  2790.               if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) {   // checker au bout de 15s
  2791.                 if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) {  // trop lent
  2792.                   back[i].status=0;  // terminΘ
  2793.                   if (back[i].r.soc!=INVALID_SOCKET) {
  2794. #if HTS_DEBUG_CLOSESOCK
  2795.                     DEBUG_W("back_wait(rateout): deletehttp\n");
  2796. #endif
  2797.                     deletehttp(&back[i].r);
  2798.                   }
  2799.                   back[i].r.soc=INVALID_SOCKET;
  2800.                   back[i].r.statuscode=-3;
  2801.                   strcpybuff(back[i].r.msg,"Transfer Rate Too Low");
  2802.                 }
  2803.               }
  2804.             }
  2805.           }
  2806.         }
  2807.       }
  2808.     }
  2809.     max_loop--;
  2810. #if HTS_ANALYSTE
  2811.     max_loop_chk++;
  2812. #endif
  2813.   } while((busy_state) && (busy_recv) && (max_loop>0));
  2814. #if HTS_ANALYSTE
  2815.   if ((!busy_recv) && (!busy_state)) {
  2816.     if (max_loop_chk>=1) {
  2817.       Sleep(10);    // un tite pause pour Θviter les lag..
  2818.     }
  2819.   }
  2820. #endif
  2821. }
  2822.  
  2823. int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize) {
  2824.   LLint size_to_test;
  2825.   if (check_only_totalsize)
  2826.     size_to_test=eback->r.totalsize;
  2827.   else
  2828.     size_to_test=max(eback->r.totalsize,eback->r.size);
  2829.   if (size_to_test>=0) {
  2830.     
  2831.     /* Interdiction taille par le wizard? */
  2832.     if (hts_testlinksize(opt,eback->url_adr,eback->url_fil,eback->r.totalsize/1024)==-1) {
  2833.       return 0;     /* interdit */
  2834.     }                     
  2835.     
  2836.     /* vΘrifier taille classique (heml et non html) */
  2837.     if ((istoobig(size_to_test,eback->maxfile_html,eback->maxfile_nonhtml,eback->r.contenttype))) {
  2838.       return 0;     /* interdit */
  2839.     }
  2840.   }
  2841.   return 1;
  2842. }
  2843.  
  2844. int back_checkmirror(httrackp* opt) {
  2845.   // Check max time
  2846.   if ((opt->maxsite>0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
  2847.     if (opt->errlog) {
  2848.       fprintf(opt->errlog,"More than "LLintP" bytes have been transfered.. giving up"LF,(LLint)opt->maxsite);
  2849.       test_flush;
  2850.     } 
  2851.     return 0;
  2852.   } else if ((opt->maxtime>0) && ((time_local()-HTS_STAT.stat_timestart)>opt->maxtime)) {            
  2853.     if (opt->errlog) {
  2854.       fprintf(opt->errlog,"More than %d seconds passed.. giving up"LF,opt->maxtime);
  2855.       test_flush;
  2856.     } 
  2857.     return 0;
  2858.   }
  2859.   return 1;   /* Ok, go on */
  2860. }
  2861.  
  2862.  
  2863. // octets transfΘrΘs + add
  2864. LLint back_transfered(LLint nb,lien_back* back,int back_max) {
  2865.   int i;
  2866.   // ajouter octets en instance
  2867.   for(i=0;i<back_max;i++)
  2868.     if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000))
  2869.       nb+=back[i].r.size;
  2870.   return nb;      
  2871. }
  2872.  
  2873. // infos backing
  2874. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  2875. void back_info(lien_back* back,int i,int j,FILE* fp) {
  2876.   if (back[i].status>=0) {
  2877.     char s[HTS_URLMAXSIZE*2+1024]; 
  2878.     s[0]='\0';
  2879.     back_infostr(back,i,j,s);
  2880.     strcatbuff(s,LF);
  2881.     fprintf(fp,"%s",s);
  2882.   }
  2883. }
  2884.  
  2885. // infos backing
  2886. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  2887. void back_infostr(lien_back* back,int i,int j,char* s) {
  2888.   if (back[i].status>=0) {
  2889.     int aff=0;
  2890.     if (j & 1) {
  2891.       if (back[i].status==100) {
  2892.         strcatbuff(s,"CONNECT ");
  2893.       } else if (back[i].status==99) {
  2894.         strcatbuff(s,"INFOS ");
  2895.         aff=1;
  2896.       } else if (back[i].status==98 || back[i].status==97) {
  2897.         strcatbuff(s,"INFOSC");             // infos chunk
  2898.         aff=1;
  2899.       }
  2900.       else if (back[i].status>0) {
  2901. #if HTS_ANALYSTE==2
  2902.         strcatbuff(s,"WAIT ");
  2903. #else
  2904.         strcatbuff(s,"RECEIVE "); 
  2905. #endif
  2906.         aff=1; 
  2907.       }
  2908.     } 
  2909.     if (j & 2) {
  2910.       if (back[i].status==0) {
  2911.         switch (back[i].r.statuscode) {
  2912.         case 200:
  2913.           strcatbuff(s,"READY ");
  2914.           aff=1;
  2915.           break;
  2916. #if HTS_ANALYSTE==2
  2917.         default:
  2918.           strcatbuff(s,"ERROR ");
  2919.           break;
  2920. #else
  2921.         case -1:
  2922.           strcatbuff(s,"ERROR ");
  2923.           aff=1;
  2924.           break;
  2925.         case -2:
  2926.           strcatbuff(s,"TIMEOUT ");
  2927.           aff=1;
  2928.           break;
  2929.         case -3:
  2930.           strcatbuff(s,"TOOSLOW ");
  2931.           aff=1;
  2932.           break;
  2933.         case 400:
  2934.           strcatbuff(s,"BADREQUEST ");
  2935.           aff=1;
  2936.           break;
  2937.         case 401: case 403:
  2938.           strcatbuff(s,"FORBIDDEN ");
  2939.           aff=1;
  2940.           break;
  2941.         case 404:
  2942.           strcatbuff(s,"NOT FOUND ");
  2943.           aff=1;
  2944.           break;
  2945.         case 500:
  2946.           strcatbuff(s,"SERVERROR ");
  2947.           aff=1;
  2948.           break;
  2949.         default:
  2950.           {
  2951.             char s2[256];
  2952.             sprintf(s2,"ERROR(%d)",back[i].r.statuscode);
  2953.             strcatbuff(s,s2);
  2954.           }
  2955.           aff=1;
  2956. #endif
  2957.         }
  2958.       }
  2959.     }
  2960.     
  2961.     if (aff) {
  2962.       {
  2963.         char s2[HTS_URLMAXSIZE*2+1024];
  2964.         sprintf(s2,"\"%s",back[i].url_adr); strcatbuff(s,s2);
  2965.         
  2966.         if (back[i].url_fil[0]!='/') strcatbuff(s,"/");
  2967.         sprintf(s2,"%s\" ",back[i].url_fil); strcatbuff(s,s2);
  2968.         sprintf(s,LLintP" "LLintP" ",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); strcatbuff(s,s2);
  2969.       }
  2970.     }
  2971.   }
  2972. }
  2973.  
  2974. // -- backing --
  2975.  
  2976. #undef test_flush
  2977.